15:07:11.738 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
15:07:11.738 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskco6SfS/test-tmp
15:07:11.738 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/debug/bin
15:07:11.738 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskco6SfS/java/build/dist-test/../../../build/debug/bin/kudu, test, mini_cluster, --serialization=pb]
15:07:11.739 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
15:07:11.850 [DEBUG - main] (MiniKuduCluster.java:195) Response:
15:07:11.851 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
15:07:11.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:11.852185 3505 mini_oidc.cc:150] Starting JWKS server
15:07:11.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:11.854226 3505 webserver.cc:458] Webserver started at http://127.0.0.1:33145/ using document root <none> and password file <none>
15:07:11.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:11.854382 3505 mini_oidc.cc:186] Starting OIDC Discovery server
15:07:11.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:11.854636 3505 webserver.cc:458] Webserver started at http://127.0.0.1:33447/ using document root <none> and password file <none>
15:07:11.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-12T15:07:11Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
15:07:11.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-12T15:07:11Z Disabled control of system clock
15:07:11.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:11.870160 3505 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskco6SfS/build/debug/bin/kudu
15:07:11.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskco6SfS/build/debug/bin/kudu
15:07:11.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/master-0/wal
15:07:11.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/master-0/data
15:07:11.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:07:11.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:07:11.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:07:11.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:07:11.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:07:11.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:07:11.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/master-0/logs
15:07:11.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/master-0/data/info.pb
15:07:11.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:07:11.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:07:11.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:07:11.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:07:11.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:07:11.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:07:11.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:07:11.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:07:11.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:07:11.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:07:11.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:07:11.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:07:11.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.108.126:33867
15:07:11.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.108.126
15:07:11.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:07:11.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.108.84:35745
15:07:11.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:07:11.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:07:11.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:07:11.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:07:11.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.108.126:33867,127.3.108.125:39569,127.3.108.124:35887
15:07:11.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:07:11.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33145/jwks.json with env {}
15:07:11.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:11.977469 3516 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:07:11.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:11.977732 3516 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:07:11.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:11.977782 3516 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:07:11.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:11.981083 3516 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:07:11.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:11.981158 3516 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:07:11.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:11.981179 3516 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:07:11.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:11.981196 3516 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:07:11.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:11.981212 3516 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:07:11.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:11.981225 3516 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33145/jwks.json
15:07:11.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:11.985224 3516 master_runner.cc:386] Master server non-default flags:
15:07:11.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:07:11.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.108.84:35745
15:07:11.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:07:11.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:07:11.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/master-0/data
15:07:11.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/master-0/wal
15:07:11.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:07:11.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.108.126:33867,127.3.108.125:39569,127.3.108.124:35887
15:07:11.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:07:11.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:07:11.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:07:11.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.108.126:33867
15:07:11.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:07:11.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:07:11.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:07:11.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33145/jwks.json
15:07:11.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:07:11.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:07:11.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/master-0/data/info.pb
15:07:11.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.108.126
15:07:11.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:07:11.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:07:11.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.3516
15:07:11.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:07:11.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:07:11.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:07:11.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:07:11.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/master-0/logs
15:07:11.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:07:11.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:07:11.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:07:11.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:07:11.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:07:11.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:07:11.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
15:07:11.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Jan 2025 14:43:14 UTC on 24a791456cd2
15:07:11.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4141
15:07:11.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:11.986320 3516 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:07:11.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:11.987457 3516 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:07:11.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:11.994896 3524 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:07:11.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:11.995558 3521 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:07:11.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:11.996181 3522 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:07:11.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:11.997020 3516 server_base.cc:1034] running on GCE node
15:07:11.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:11.997606 3516 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:07:11.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:11.998623 3516 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:07:11.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:11.999784 3516 hybrid_clock.cc:648] HybridClock initialized: now 1736694431999772 us; error 37 us; skew 500 ppm
15:07:12.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.001852 3516 webserver.cc:458] Webserver started at http://127.3.108.126:33641/ using document root <none> and password file <none>
15:07:12.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.002403 3516 fs_manager.cc:362] Metadata directory not provided
15:07:12.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.002465 3516 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:07:12.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.002624 3516 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:07:12.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.004436 3516 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/master-0/data/instance:
15:07:12.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8ad36199f9534c2db694d8d704696ff3"
15:07:12.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:07:11 on dist-test-slave-kjk3"
15:07:12.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.004956 3516 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/master-0/wal/instance:
15:07:12.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8ad36199f9534c2db694d8d704696ff3"
15:07:12.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:07:11 on dist-test-slave-kjk3"
15:07:12.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.008473 3516 fs_manager.cc:696] Time spent creating directory manager: real 0.003s user 0.000s sys 0.003s
15:07:12.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.010774 3530 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:07:12.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.011914 3516 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
15:07:12.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.012028 3516 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/master-0/data,/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/master-0/wal
15:07:12.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8ad36199f9534c2db694d8d704696ff3"
15:07:12.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:07:11 on dist-test-slave-kjk3"
15:07:12.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.012151 3516 fs_report.cc:389] FS layout report
15:07:12.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:07:12.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/master-0/wal
15:07:12.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/master-0/wal
15:07:12.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/master-0/data/data
15:07:12.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:07:12.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:07:12.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:07:12.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:07:12.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:07:12.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:07:12.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:07:12.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:07:12.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:07:12.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:07:12.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:07:12.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.027303 3516 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:07:12.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.028203 3516 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:07:12.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.028380 3516 kserver.cc:163] Server-wide thread pool size limit: 3276
15:07:12.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.035256 3516 rpc_server.cc:307] RPC server started. Bound to: 127.3.108.126:33867
15:07:12.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.035261 3582 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.3.108.126:33867 every 8 connection(s)
15:07:12.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.036409 3516 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/master-0/data/info.pb
15:07:12.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.039043 3583 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:07:12.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.042757 3583 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.108.126" port: 33867 } has no permanent_uuid. Determining permanent_uuid...
15:07:12.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.046188 3505 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskco6SfS/build/debug/bin/kudu as pid 3516
15:07:12.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.046344 3505 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/master-0/wal/instance
15:07:12.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.047945 3505 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskco6SfS/build/debug/bin/kudu
15:07:12.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskco6SfS/build/debug/bin/kudu
15:07:12.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/master-1/wal
15:07:12.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/master-1/data
15:07:12.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:07:12.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:07:12.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:07:12.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:07:12.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:07:12.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:07:12.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/master-1/logs
15:07:12.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/master-1/data/info.pb
15:07:12.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:07:12.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:07:12.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:07:12.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:07:12.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:07:12.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:07:12.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:07:12.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:07:12.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:07:12.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:07:12.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:07:12.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:07:12.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.108.125:39569
15:07:12.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.108.125
15:07:12.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:07:12.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.108.84:35745
15:07:12.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:07:12.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:07:12.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:07:12.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:07:12.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.108.126:33867,127.3.108.125:39569,127.3.108.124:35887
15:07:12.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:07:12.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33145/jwks.json with env {}
15:07:12.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.055191 3583 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.108.125" port: 39569 } has no permanent_uuid. Determining permanent_uuid...
15:07:12.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:12.056172 3533 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.108.125:39569: connect: Connection refused (error 111)
15:07:12.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:12.057166 3583 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.108.125:39569: Network error: Client connection negotiation failed: client connection to 127.3.108.125:39569: connect: Connection refused (error 111)
15:07:12.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.106420 3583 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.108.125" port: 39569 } attempt: 1
15:07:12.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:12.108019 3583 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.108.125:39569: Network error: Client connection negotiation failed: client connection to 127.3.108.125:39569: connect: Connection refused (error 111)
15:07:12.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:12.158775 3587 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:07:12.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:12.159029 3587 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:07:12.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:12.159096 3587 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:07:12.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:12.162426 3587 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:07:12.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:12.162504 3587 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:07:12.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:12.162532 3587 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:07:12.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:12.162556 3587 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:07:12.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:12.162580 3587 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:07:12.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:12.162597 3587 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33145/jwks.json
15:07:12.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.166628 3587 master_runner.cc:386] Master server non-default flags:
15:07:12.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:07:12.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.108.84:35745
15:07:12.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:07:12.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:07:12.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/master-1/data
15:07:12.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/master-1/wal
15:07:12.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:07:12.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.108.126:33867,127.3.108.125:39569,127.3.108.124:35887
15:07:12.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:07:12.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:07:12.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:07:12.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.108.125:39569
15:07:12.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:07:12.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:07:12.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:07:12.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33145/jwks.json
15:07:12.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:07:12.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:07:12.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/master-1/data/info.pb
15:07:12.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.108.125
15:07:12.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:07:12.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:07:12.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.3587
15:07:12.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:07:12.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:07:12.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:07:12.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:07:12.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/master-1/logs
15:07:12.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:07:12.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:07:12.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:07:12.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:07:12.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:07:12.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:07:12.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
15:07:12.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Jan 2025 14:43:14 UTC on 24a791456cd2
15:07:12.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4141
15:07:12.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.168016 3587 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:07:12.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.169226 3587 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:07:12.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.176174 3583 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.108.125" port: 39569 } attempt: 2
15:07:12.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:12.176579 3594 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:07:12.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:12.177106 3596 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:07:12.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:12.177556 3583 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.108.125:39569: Network error: Client connection negotiation failed: client connection to 127.3.108.125:39569: connect: Connection refused (error 111)
15:07:12.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:12.177896 3593 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:07:12.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.178643 3587 server_base.cc:1034] running on GCE node
15:07:12.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.179239 3587 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:07:12.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.180277 3587 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:07:12.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.181476 3587 hybrid_clock.cc:648] HybridClock initialized: now 1736694432181463 us; error 28 us; skew 500 ppm
15:07:12.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.183559 3587 webserver.cc:458] Webserver started at http://127.3.108.125:38841/ using document root <none> and password file <none>
15:07:12.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.184223 3587 fs_manager.cc:362] Metadata directory not provided
15:07:12.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.184291 3587 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:07:12.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.184537 3587 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:07:12.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.186304 3587 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/master-1/data/instance:
15:07:12.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "80dd9bfdcef247a9a343c37e9c1aa547"
15:07:12.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:07:12 on dist-test-slave-kjk3"
15:07:12.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.186836 3587 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/master-1/wal/instance:
15:07:12.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "80dd9bfdcef247a9a343c37e9c1aa547"
15:07:12.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:07:12 on dist-test-slave-kjk3"
15:07:12.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.190536 3587 fs_manager.cc:696] Time spent creating directory manager: real 0.003s user 0.005s sys 0.000s
15:07:12.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.193102 3602 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:07:12.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.194285 3587 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.003s sys 0.000s
15:07:12.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.194442 3587 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/master-1/data,/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/master-1/wal
15:07:12.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "80dd9bfdcef247a9a343c37e9c1aa547"
15:07:12.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:07:12 on dist-test-slave-kjk3"
15:07:12.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.194535 3587 fs_report.cc:389] FS layout report
15:07:12.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:07:12.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/master-1/wal
15:07:12.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/master-1/wal
15:07:12.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/master-1/data/data
15:07:12.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:07:12.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:07:12.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:07:12.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:07:12.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:07:12.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:07:12.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:07:12.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:07:12.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:07:12.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:07:12.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:07:12.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.206146 3587 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:07:12.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.206815 3587 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:07:12.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.206959 3587 kserver.cc:163] Server-wide thread pool size limit: 3276
15:07:12.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.213732 3587 rpc_server.cc:307] RPC server started. Bound to: 127.3.108.125:39569
15:07:12.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.213742 3654 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.3.108.125:39569 every 8 connection(s)
15:07:12.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.214851 3587 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/master-1/data/info.pb
15:07:12.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.217828 3655 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:07:12.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.221798 3655 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.108.126" port: 33867 } has no permanent_uuid. Determining permanent_uuid...
15:07:12.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.224956 3505 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskco6SfS/build/debug/bin/kudu as pid 3587
15:07:12.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.225062 3505 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/master-1/wal/instance
15:07:12.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.225641 3505 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskco6SfS/build/debug/bin/kudu
15:07:12.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskco6SfS/build/debug/bin/kudu
15:07:12.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/master-2/wal
15:07:12.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/master-2/data
15:07:12.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:07:12.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:07:12.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:07:12.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:07:12.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:07:12.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:07:12.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/master-2/logs
15:07:12.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/master-2/data/info.pb
15:07:12.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:07:12.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:07:12.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:07:12.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:07:12.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:07:12.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:07:12.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:07:12.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:07:12.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:07:12.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:07:12.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:07:12.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:07:12.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.108.124:35887
15:07:12.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.108.124
15:07:12.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:07:12.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.108.84:35745
15:07:12.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:07:12.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:07:12.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:07:12.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:07:12.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.108.126:33867,127.3.108.125:39569,127.3.108.124:35887
15:07:12.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:07:12.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33145/jwks.json with env {}
15:07:12.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.231069 3655 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.108.125" port: 39569 } has no permanent_uuid. Determining permanent_uuid...
15:07:12.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.237066 3655 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.108.124" port: 35887 } has no permanent_uuid. Determining permanent_uuid...
15:07:12.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:12.237793 3606 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.108.124:35887: connect: Connection refused (error 111)
15:07:12.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:12.238672 3655 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.108.124:35887: Network error: Client connection negotiation failed: client connection to 127.3.108.124:35887: connect: Connection refused (error 111)
15:07:12.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.268709 3583 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.108.125" port: 39569 } attempt: 3
15:07:12.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.273749 3583 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.108.124" port: 35887 } has no permanent_uuid. Determining permanent_uuid...
15:07:12.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:12.275063 3583 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.108.124:35887: Network error: Client connection negotiation failed: client connection to 127.3.108.124:35887: connect: Connection refused (error 111)
15:07:12.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.287884 3655 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.108.124" port: 35887 } attempt: 1
15:07:12.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:12.289340 3655 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.108.124:35887: Network error: Client connection negotiation failed: client connection to 127.3.108.124:35887: connect: Connection refused (error 111)
15:07:12.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.306258 3583 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.108.124" port: 35887 } attempt: 1
15:07:12.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:12.307885 3583 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.108.124:35887: Network error: Client connection negotiation failed: client connection to 127.3.108.124:35887: connect: Connection refused (error 111)
15:07:12.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:12.349696 3658 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:07:12.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:12.349939 3658 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:07:12.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:12.349987 3658 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:07:12.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:12.353156 3658 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:07:12.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:12.353224 3658 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:07:12.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:12.353242 3658 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:07:12.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:12.353260 3658 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:07:12.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:12.353276 3658 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:07:12.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:12.353289 3658 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33145/jwks.json
15:07:12.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.357175 3658 master_runner.cc:386] Master server non-default flags:
15:07:12.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:07:12.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.108.84:35745
15:07:12.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:07:12.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:07:12.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/master-2/data
15:07:12.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/master-2/wal
15:07:12.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:07:12.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.108.126:33867,127.3.108.125:39569,127.3.108.124:35887
15:07:12.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:07:12.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:07:12.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:07:12.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.108.124:35887
15:07:12.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:07:12.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:07:12.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:07:12.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33145/jwks.json
15:07:12.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:07:12.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:07:12.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/master-2/data/info.pb
15:07:12.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.108.124
15:07:12.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:07:12.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:07:12.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.3658
15:07:12.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:07:12.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:07:12.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:07:12.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:07:12.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/master-2/logs
15:07:12.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:07:12.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:07:12.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:07:12.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:07:12.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:07:12.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:07:12.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
15:07:12.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Jan 2025 14:43:14 UTC on 24a791456cd2
15:07:12.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4141
15:07:12.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.357518 3655 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.108.124" port: 35887 } attempt: 2
15:07:12.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.358421 3658 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:07:12.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:12.359150 3655 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.108.124:35887: Network error: Client connection negotiation failed: client connection to 127.3.108.124:35887: connect: Connection refused (error 111)
15:07:12.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.359767 3658 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:07:12.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:12.367013 3666 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:07:12.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:12.367655 3668 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:07:12.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:12.368252 3665 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:07:12.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.369061 3658 server_base.cc:1034] running on GCE node
15:07:12.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.369691 3658 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:07:12.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.370712 3658 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:07:12.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.371899 3658 hybrid_clock.cc:648] HybridClock initialized: now 1736694432371879 us; error 33 us; skew 500 ppm
15:07:12.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.373945 3658 webserver.cc:458] Webserver started at http://127.3.108.124:34115/ using document root <none> and password file <none>
15:07:12.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.374508 3658 fs_manager.cc:362] Metadata directory not provided
15:07:12.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.374558 3658 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:07:12.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.374708 3658 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:07:12.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.376425 3658 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/master-2/data/instance:
15:07:12.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8fa5c38def8046f5a1a8589e2ec31711"
15:07:12.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:07:12 on dist-test-slave-kjk3"
15:07:12.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.376874 3658 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/master-2/wal/instance:
15:07:12.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8fa5c38def8046f5a1a8589e2ec31711"
15:07:12.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:07:12 on dist-test-slave-kjk3"
15:07:12.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.380349 3658 fs_manager.cc:696] Time spent creating directory manager: real 0.003s user 0.003s sys 0.001s
15:07:12.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.382627 3674 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:07:12.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.383059 3583 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.108.124" port: 35887 } attempt: 2
15:07:12.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.383893 3658 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
15:07:12.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.384014 3658 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/master-2/data,/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/master-2/wal
15:07:12.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8fa5c38def8046f5a1a8589e2ec31711"
15:07:12.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:07:12 on dist-test-slave-kjk3"
15:07:12.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.384143 3658 fs_report.cc:389] FS layout report
15:07:12.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:07:12.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/master-2/wal
15:07:12.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/master-2/wal
15:07:12.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/master-2/data/data
15:07:12.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:07:12.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:07:12.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:07:12.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:07:12.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:07:12.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:07:12.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:07:12.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:07:12.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:07:12.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:07:12.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:07:12.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:12.384532 3583 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.108.124:35887: Network error: Client connection negotiation failed: client connection to 127.3.108.124:35887: connect: Connection refused (error 111)
15:07:12.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.398365 3658 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:07:12.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.399061 3658 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:07:12.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.399221 3658 kserver.cc:163] Server-wide thread pool size limit: 3276
15:07:12.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.406160 3658 rpc_server.cc:307] RPC server started. Bound to: 127.3.108.124:35887
15:07:12.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.406176 3726 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.3.108.124:35887 every 8 connection(s)
15:07:12.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.407151 3658 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/master-2/data/info.pb
15:07:12.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.409821 3727 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:07:12.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.411581 3505 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskco6SfS/build/debug/bin/kudu as pid 3658
15:07:12.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.411693 3505 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/master-2/wal/instance
15:07:12.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.413892 3727 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.108.126" port: 33867 } has no permanent_uuid. Determining permanent_uuid...
15:07:12.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.423398 3727 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.108.125" port: 39569 } has no permanent_uuid. Determining permanent_uuid...
15:07:12.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.427762 3727 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.108.124" port: 35887 } has no permanent_uuid. Determining permanent_uuid...
15:07:12.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.433501 3727 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8fa5c38def8046f5a1a8589e2ec31711: Bootstrap starting.
15:07:12.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.435766 3727 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 8fa5c38def8046f5a1a8589e2ec31711: Neither blocks nor log segments found. Creating new log.
15:07:12.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.437026 3727 log.cc:826] T 00000000000000000000000000000000 P 8fa5c38def8046f5a1a8589e2ec31711: Log is configured to *not* fsync() on all Append() calls
15:07:12.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.438738 3727 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8fa5c38def8046f5a1a8589e2ec31711: No bootstrap required, opened a new log
15:07:12.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.441614 3727 raft_consensus.cc:357] T 00000000000000000000000000000000 P 8fa5c38def8046f5a1a8589e2ec31711 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ad36199f9534c2db694d8d704696ff3" member_type: VOTER last_known_addr { host: "127.3.108.126" port: 33867 } } peers { permanent_uuid: "80dd9bfdcef247a9a343c37e9c1aa547" member_type: VOTER last_known_addr { host: "127.3.108.125" port: 39569 } } peers { permanent_uuid: "8fa5c38def8046f5a1a8589e2ec31711" member_type: VOTER last_known_addr { host: "127.3.108.124" port: 35887 } }
15:07:12.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.441797 3727 raft_consensus.cc:383] T 00000000000000000000000000000000 P 8fa5c38def8046f5a1a8589e2ec31711 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:07:12.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.441831 3727 raft_consensus.cc:738] T 00000000000000000000000000000000 P 8fa5c38def8046f5a1a8589e2ec31711 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8fa5c38def8046f5a1a8589e2ec31711, State: Initialized, Role: FOLLOWER
15:07:12.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.442225 3727 consensus_queue.cc:260] T 00000000000000000000000000000000 P 8fa5c38def8046f5a1a8589e2ec31711 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ad36199f9534c2db694d8d704696ff3" member_type: VOTER last_known_addr { host: "127.3.108.126" port: 33867 } } peers { permanent_uuid: "80dd9bfdcef247a9a343c37e9c1aa547" member_type: VOTER last_known_addr { host: "127.3.108.125" port: 39569 } } peers { permanent_uuid: "8fa5c38def8046f5a1a8589e2ec31711" member_type: VOTER last_known_addr { host: "127.3.108.124" port: 35887 } }
15:07:12.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.442669 3736 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8fa5c38def8046f5a1a8589e2ec31711 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ad36199f9534c2db694d8d704696ff3" member_type: VOTER last_known_addr { host: "127.3.108.126" port: 33867 } } peers { permanent_uuid: "80dd9bfdcef247a9a343c37e9c1aa547" member_type: VOTER last_known_addr { host: "127.3.108.125" port: 39569 } } peers { permanent_uuid: "8fa5c38def8046f5a1a8589e2ec31711" member_type: VOTER last_known_addr { host: "127.3.108.124" port: 35887 } } }
15:07:12.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.442786 3736 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8fa5c38def8046f5a1a8589e2ec31711 [sys.catalog]: This master's current role is: FOLLOWER
15:07:12.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.443360 3727 sys_catalog.cc:564] T 00000000000000000000000000000000 P 8fa5c38def8046f5a1a8589e2ec31711 [sys.catalog]: configured and running, proceeding with master startup.
15:07:12.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:12.448447 3747 catalog_manager.cc:1559] T 00000000000000000000000000000000 P 8fa5c38def8046f5a1a8589e2ec31711: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:07:12.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:12.448536 3747 catalog_manager.cc:874] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:07:12.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.446660 3658 master_runner.cc:186] Error getting master registration for 127.3.108.126:33867: OK, instance_id { permanent_uuid: "8ad36199f9534c2db694d8d704696ff3" instance_seqno: 1736694432033278 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
15:07:12.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.448673 3658 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:07:12.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.450383 3655 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.108.124" port: 35887 } attempt: 3
15:07:12.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.456784 3655 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 80dd9bfdcef247a9a343c37e9c1aa547: Bootstrap starting.
15:07:12.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.459254 3655 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 80dd9bfdcef247a9a343c37e9c1aa547: Neither blocks nor log segments found. Creating new log.
15:07:12.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.460053 3655 log.cc:826] T 00000000000000000000000000000000 P 80dd9bfdcef247a9a343c37e9c1aa547: Log is configured to *not* fsync() on all Append() calls
15:07:12.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.462060 3655 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 80dd9bfdcef247a9a343c37e9c1aa547: No bootstrap required, opened a new log
15:07:12.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.464830 3655 raft_consensus.cc:357] T 00000000000000000000000000000000 P 80dd9bfdcef247a9a343c37e9c1aa547 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ad36199f9534c2db694d8d704696ff3" member_type: VOTER last_known_addr { host: "127.3.108.126" port: 33867 } } peers { permanent_uuid: "80dd9bfdcef247a9a343c37e9c1aa547" member_type: VOTER last_known_addr { host: "127.3.108.125" port: 39569 } } peers { permanent_uuid: "8fa5c38def8046f5a1a8589e2ec31711" member_type: VOTER last_known_addr { host: "127.3.108.124" port: 35887 } }
15:07:12.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.464990 3655 raft_consensus.cc:383] T 00000000000000000000000000000000 P 80dd9bfdcef247a9a343c37e9c1aa547 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:07:12.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.465030 3655 raft_consensus.cc:738] T 00000000000000000000000000000000 P 80dd9bfdcef247a9a343c37e9c1aa547 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 80dd9bfdcef247a9a343c37e9c1aa547, State: Initialized, Role: FOLLOWER
15:07:12.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.465431 3655 consensus_queue.cc:260] T 00000000000000000000000000000000 P 80dd9bfdcef247a9a343c37e9c1aa547 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ad36199f9534c2db694d8d704696ff3" member_type: VOTER last_known_addr { host: "127.3.108.126" port: 33867 } } peers { permanent_uuid: "80dd9bfdcef247a9a343c37e9c1aa547" member_type: VOTER last_known_addr { host: "127.3.108.125" port: 39569 } } peers { permanent_uuid: "8fa5c38def8046f5a1a8589e2ec31711" member_type: VOTER last_known_addr { host: "127.3.108.124" port: 35887 } }
15:07:12.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.465904 3750 sys_catalog.cc:455] T 00000000000000000000000000000000 P 80dd9bfdcef247a9a343c37e9c1aa547 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ad36199f9534c2db694d8d704696ff3" member_type: VOTER last_known_addr { host: "127.3.108.126" port: 33867 } } peers { permanent_uuid: "80dd9bfdcef247a9a343c37e9c1aa547" member_type: VOTER last_known_addr { host: "127.3.108.125" port: 39569 } } peers { permanent_uuid: "8fa5c38def8046f5a1a8589e2ec31711" member_type: VOTER last_known_addr { host: "127.3.108.124" port: 35887 } } }
15:07:12.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.466017 3750 sys_catalog.cc:458] T 00000000000000000000000000000000 P 80dd9bfdcef247a9a343c37e9c1aa547 [sys.catalog]: This master's current role is: FOLLOWER
15:07:12.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.466612 3655 sys_catalog.cc:564] T 00000000000000000000000000000000 P 80dd9bfdcef247a9a343c37e9c1aa547 [sys.catalog]: configured and running, proceeding with master startup.
15:07:12.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:12.472247 3761 catalog_manager.cc:1559] T 00000000000000000000000000000000 P 80dd9bfdcef247a9a343c37e9c1aa547: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:07:12.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:12.472524 3761 catalog_manager.cc:874] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:07:12.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.470640 3587 master_runner.cc:186] Error getting master registration for 127.3.108.126:33867: OK, instance_id { permanent_uuid: "8ad36199f9534c2db694d8d704696ff3" instance_seqno: 1736694432033278 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
15:07:12.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.473492 3587 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:07:12.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.481150 3736 raft_consensus.cc:491] T 00000000000000000000000000000000 P 8fa5c38def8046f5a1a8589e2ec31711 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:07:12.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.481374 3736 raft_consensus.cc:513] T 00000000000000000000000000000000 P 8fa5c38def8046f5a1a8589e2ec31711 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ad36199f9534c2db694d8d704696ff3" member_type: VOTER last_known_addr { host: "127.3.108.126" port: 33867 } } peers { permanent_uuid: "80dd9bfdcef247a9a343c37e9c1aa547" member_type: VOTER last_known_addr { host: "127.3.108.125" port: 39569 } } peers { permanent_uuid: "8fa5c38def8046f5a1a8589e2ec31711" member_type: VOTER last_known_addr { host: "127.3.108.124" port: 35887 } }
15:07:12.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.482471 3736 leader_election.cc:290] T 00000000000000000000000000000000 P 8fa5c38def8046f5a1a8589e2ec31711 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8ad36199f9534c2db694d8d704696ff3 (127.3.108.126:33867), 80dd9bfdcef247a9a343c37e9c1aa547 (127.3.108.125:39569)
15:07:12.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.483065 3629 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "8fa5c38def8046f5a1a8589e2ec31711" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "80dd9bfdcef247a9a343c37e9c1aa547" is_pre_election: true
15:07:12.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.483419 3629 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 80dd9bfdcef247a9a343c37e9c1aa547 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8fa5c38def8046f5a1a8589e2ec31711 in term 0.
15:07:12.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.483673 3583 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.108.124" port: 35887 } attempt: 3
15:07:12.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.483860 3677 leader_election.cc:304] T 00000000000000000000000000000000 P 8fa5c38def8046f5a1a8589e2ec31711 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 80dd9bfdcef247a9a343c37e9c1aa547, 8fa5c38def8046f5a1a8589e2ec31711; no voters:
15:07:12.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.484166 3736 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 8fa5c38def8046f5a1a8589e2ec31711 [term 0 FOLLOWER]: Leader pre-election won for term 1
15:07:12.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.484241 3736 raft_consensus.cc:491] T 00000000000000000000000000000000 P 8fa5c38def8046f5a1a8589e2ec31711 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
15:07:12.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.484282 3736 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 8fa5c38def8046f5a1a8589e2ec31711 [term 0 FOLLOWER]: Advancing to term 1
15:07:12.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.485397 3736 raft_consensus.cc:513] T 00000000000000000000000000000000 P 8fa5c38def8046f5a1a8589e2ec31711 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ad36199f9534c2db694d8d704696ff3" member_type: VOTER last_known_addr { host: "127.3.108.126" port: 33867 } } peers { permanent_uuid: "80dd9bfdcef247a9a343c37e9c1aa547" member_type: VOTER last_known_addr { host: "127.3.108.125" port: 39569 } } peers { permanent_uuid: "8fa5c38def8046f5a1a8589e2ec31711" member_type: VOTER last_known_addr { host: "127.3.108.124" port: 35887 } }
15:07:12.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.485761 3736 leader_election.cc:290] T 00000000000000000000000000000000 P 8fa5c38def8046f5a1a8589e2ec31711 [CANDIDATE]: Term 1 election: Requested vote from peers 8ad36199f9534c2db694d8d704696ff3 (127.3.108.126:33867), 80dd9bfdcef247a9a343c37e9c1aa547 (127.3.108.125:39569)
15:07:12.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.486670 3629 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "8fa5c38def8046f5a1a8589e2ec31711" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "80dd9bfdcef247a9a343c37e9c1aa547"
15:07:12.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.486790 3629 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 80dd9bfdcef247a9a343c37e9c1aa547 [term 0 FOLLOWER]: Advancing to term 1
15:07:12.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.487972 3629 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 80dd9bfdcef247a9a343c37e9c1aa547 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8fa5c38def8046f5a1a8589e2ec31711 in term 1.
15:07:12.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.488361 3677 leader_election.cc:304] T 00000000000000000000000000000000 P 8fa5c38def8046f5a1a8589e2ec31711 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 80dd9bfdcef247a9a343c37e9c1aa547, 8fa5c38def8046f5a1a8589e2ec31711; no voters:
15:07:12.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.489728 3736 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 8fa5c38def8046f5a1a8589e2ec31711 [term 1 FOLLOWER]: Leader election won for term 1
15:07:12.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.490182 3736 raft_consensus.cc:695] T 00000000000000000000000000000000 P 8fa5c38def8046f5a1a8589e2ec31711 [term 1 LEADER]: Becoming Leader. State: Replica: 8fa5c38def8046f5a1a8589e2ec31711, State: Running, Role: LEADER
15:07:12.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.490818 3736 consensus_queue.cc:237] T 00000000000000000000000000000000 P 8fa5c38def8046f5a1a8589e2ec31711 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ad36199f9534c2db694d8d704696ff3" member_type: VOTER last_known_addr { host: "127.3.108.126" port: 33867 } } peers { permanent_uuid: "80dd9bfdcef247a9a343c37e9c1aa547" member_type: VOTER last_known_addr { host: "127.3.108.125" port: 39569 } } peers { permanent_uuid: "8fa5c38def8046f5a1a8589e2ec31711" member_type: VOTER last_known_addr { host: "127.3.108.124" port: 35887 } }
15:07:12.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.491020 3583 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8ad36199f9534c2db694d8d704696ff3: Bootstrap starting.
15:07:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.493889 3736 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8fa5c38def8046f5a1a8589e2ec31711 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 8fa5c38def8046f5a1a8589e2ec31711. Latest consensus state: current_term: 1 leader_uuid: "8fa5c38def8046f5a1a8589e2ec31711" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ad36199f9534c2db694d8d704696ff3" member_type: VOTER last_known_addr { host: "127.3.108.126" port: 33867 } } peers { permanent_uuid: "80dd9bfdcef247a9a343c37e9c1aa547" member_type: VOTER last_known_addr { host: "127.3.108.125" port: 39569 } } peers { permanent_uuid: "8fa5c38def8046f5a1a8589e2ec31711" member_type: VOTER last_known_addr { host: "127.3.108.124" port: 35887 } } }
15:07:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.494114 3583 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 8ad36199f9534c2db694d8d704696ff3: Neither blocks nor log segments found. Creating new log.
15:07:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.494242 3736 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8fa5c38def8046f5a1a8589e2ec31711 [sys.catalog]: This master's current role is: LEADER
15:07:12.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.494961 3765 catalog_manager.cc:1476] Loading table and tablet metadata into memory...
15:07:12.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.495007 3583 log.cc:826] T 00000000000000000000000000000000 P 8ad36199f9534c2db694d8d704696ff3: Log is configured to *not* fsync() on all Append() calls
15:07:12.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.496961 3765 catalog_manager.cc:1485] Initializing Kudu cluster ID...
15:07:12.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.500514 3583 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8ad36199f9534c2db694d8d704696ff3: No bootstrap required, opened a new log
15:07:12.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.506067 3583 raft_consensus.cc:357] T 00000000000000000000000000000000 P 8ad36199f9534c2db694d8d704696ff3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ad36199f9534c2db694d8d704696ff3" member_type: VOTER last_known_addr { host: "127.3.108.126" port: 33867 } } peers { permanent_uuid: "80dd9bfdcef247a9a343c37e9c1aa547" member_type: VOTER last_known_addr { host: "127.3.108.125" port: 39569 } } peers { permanent_uuid: "8fa5c38def8046f5a1a8589e2ec31711" member_type: VOTER last_known_addr { host: "127.3.108.124" port: 35887 } }
15:07:12.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.506384 3583 raft_consensus.cc:383] T 00000000000000000000000000000000 P 8ad36199f9534c2db694d8d704696ff3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:07:12.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.506465 3583 raft_consensus.cc:738] T 00000000000000000000000000000000 P 8ad36199f9534c2db694d8d704696ff3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8ad36199f9534c2db694d8d704696ff3, State: Initialized, Role: FOLLOWER
15:07:12.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.506584 3629 raft_consensus.cc:1270] T 00000000000000000000000000000000 P 80dd9bfdcef247a9a343c37e9c1aa547 [term 1 FOLLOWER]: Refusing update from remote peer 8fa5c38def8046f5a1a8589e2ec31711: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
15:07:12.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.507249 3583 consensus_queue.cc:260] T 00000000000000000000000000000000 P 8ad36199f9534c2db694d8d704696ff3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ad36199f9534c2db694d8d704696ff3" member_type: VOTER last_known_addr { host: "127.3.108.126" port: 33867 } } peers { permanent_uuid: "80dd9bfdcef247a9a343c37e9c1aa547" member_type: VOTER last_known_addr { host: "127.3.108.125" port: 39569 } } peers { permanent_uuid: "8fa5c38def8046f5a1a8589e2ec31711" member_type: VOTER last_known_addr { host: "127.3.108.124" port: 35887 } }
15:07:12.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.507846 3763 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 8fa5c38def8046f5a1a8589e2ec31711 [LEADER]: Connected to new peer: Peer: permanent_uuid: "80dd9bfdcef247a9a343c37e9c1aa547" member_type: VOTER last_known_addr { host: "127.3.108.125" port: 39569 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
15:07:12.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.509665 3556 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "8fa5c38def8046f5a1a8589e2ec31711" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8ad36199f9534c2db694d8d704696ff3" is_pre_election: true
15:07:12.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.509934 3556 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 8ad36199f9534c2db694d8d704696ff3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8fa5c38def8046f5a1a8589e2ec31711 in term 0.
15:07:12.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.512567 3750 sys_catalog.cc:455] T 00000000000000000000000000000000 P 80dd9bfdcef247a9a343c37e9c1aa547 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 8fa5c38def8046f5a1a8589e2ec31711. Latest consensus state: current_term: 1 leader_uuid: "8fa5c38def8046f5a1a8589e2ec31711" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ad36199f9534c2db694d8d704696ff3" member_type: VOTER last_known_addr { host: "127.3.108.126" port: 33867 } } peers { permanent_uuid: "80dd9bfdcef247a9a343c37e9c1aa547" member_type: VOTER last_known_addr { host: "127.3.108.125" port: 39569 } } peers { permanent_uuid: "8fa5c38def8046f5a1a8589e2ec31711" member_type: VOTER last_known_addr { host: "127.3.108.124" port: 35887 } } }
15:07:12.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.512670 3750 sys_catalog.cc:458] T 00000000000000000000000000000000 P 80dd9bfdcef247a9a343c37e9c1aa547 [sys.catalog]: This master's current role is: FOLLOWER
15:07:12.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.513103 3557 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "8fa5c38def8046f5a1a8589e2ec31711" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8ad36199f9534c2db694d8d704696ff3"
15:07:12.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.513221 3557 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 8ad36199f9534c2db694d8d704696ff3 [term 0 FOLLOWER]: Advancing to term 1
15:07:12.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.513406 3583 sys_catalog.cc:564] T 00000000000000000000000000000000 P 8ad36199f9534c2db694d8d704696ff3 [sys.catalog]: configured and running, proceeding with master startup.
15:07:12.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.514393 3557 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 8ad36199f9534c2db694d8d704696ff3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8fa5c38def8046f5a1a8589e2ec31711 in term 1.
15:07:12.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.516760 3557 raft_consensus.cc:1270] T 00000000000000000000000000000000 P 8ad36199f9534c2db694d8d704696ff3 [term 1 FOLLOWER]: Refusing update from remote peer 8fa5c38def8046f5a1a8589e2ec31711: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
15:07:12.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.517284 3763 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 8fa5c38def8046f5a1a8589e2ec31711 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8ad36199f9534c2db694d8d704696ff3" member_type: VOTER last_known_addr { host: "127.3.108.126" port: 33867 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
15:07:12.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:12.518963 3781 catalog_manager.cc:1559] T 00000000000000000000000000000000 P 8ad36199f9534c2db694d8d704696ff3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:07:12.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:12.519349 3781 catalog_manager.cc:874] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:07:12.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.526687 3783 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8ad36199f9534c2db694d8d704696ff3 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 leader_uuid: "8fa5c38def8046f5a1a8589e2ec31711" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ad36199f9534c2db694d8d704696ff3" member_type: VOTER last_known_addr { host: "127.3.108.126" port: 33867 } } peers { permanent_uuid: "80dd9bfdcef247a9a343c37e9c1aa547" member_type: VOTER last_known_addr { host: "127.3.108.125" port: 39569 } } peers { permanent_uuid: "8fa5c38def8046f5a1a8589e2ec31711" member_type: VOTER last_known_addr { host: "127.3.108.124" port: 35887 } } }
15:07:12.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.528533 3763 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8fa5c38def8046f5a1a8589e2ec31711 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "8fa5c38def8046f5a1a8589e2ec31711" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ad36199f9534c2db694d8d704696ff3" member_type: VOTER last_known_addr { host: "127.3.108.126" port: 33867 } } peers { permanent_uuid: "80dd9bfdcef247a9a343c37e9c1aa547" member_type: VOTER last_known_addr { host: "127.3.108.125" port: 39569 } } peers { permanent_uuid: "8fa5c38def8046f5a1a8589e2ec31711" member_type: VOTER last_known_addr { host: "127.3.108.124" port: 35887 } } }
15:07:12.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.528704 3763 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8fa5c38def8046f5a1a8589e2ec31711 [sys.catalog]: This master's current role is: LEADER
15:07:12.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.530571 3767 mvcc.cc:204] Tried to move back new op lower bound from 7113500395537989632 to 7113500395488546816. Current Snapshot: MvccSnapshot[applied={T|T < 7113500395537989632}]
15:07:12.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:12.529875 3536 tablet.cc:2367] T 00000000000000000000000000000000 P 8ad36199f9534c2db694d8d704696ff3: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:07:12.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.531688 3783 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8ad36199f9534c2db694d8d704696ff3 [sys.catalog]: This master's current role is: FOLLOWER
15:07:12.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.531909 3783 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8ad36199f9534c2db694d8d704696ff3 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 8fa5c38def8046f5a1a8589e2ec31711. Latest consensus state: current_term: 1 leader_uuid: "8fa5c38def8046f5a1a8589e2ec31711" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ad36199f9534c2db694d8d704696ff3" member_type: VOTER last_known_addr { host: "127.3.108.126" port: 33867 } } peers { permanent_uuid: "80dd9bfdcef247a9a343c37e9c1aa547" member_type: VOTER last_known_addr { host: "127.3.108.125" port: 39569 } } peers { permanent_uuid: "8fa5c38def8046f5a1a8589e2ec31711" member_type: VOTER last_known_addr { host: "127.3.108.124" port: 35887 } } }
15:07:12.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.532899 3736 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8fa5c38def8046f5a1a8589e2ec31711 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "8fa5c38def8046f5a1a8589e2ec31711" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ad36199f9534c2db694d8d704696ff3" member_type: VOTER last_known_addr { host: "127.3.108.126" port: 33867 } } peers { permanent_uuid: "80dd9bfdcef247a9a343c37e9c1aa547" member_type: VOTER last_known_addr { host: "127.3.108.125" port: 39569 } } peers { permanent_uuid: "8fa5c38def8046f5a1a8589e2ec31711" member_type: VOTER last_known_addr { host: "127.3.108.124" port: 35887 } } }
15:07:12.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.533020 3736 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8fa5c38def8046f5a1a8589e2ec31711 [sys.catalog]: This master's current role is: LEADER
15:07:12.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.534956 3750 sys_catalog.cc:455] T 00000000000000000000000000000000 P 80dd9bfdcef247a9a343c37e9c1aa547 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "8fa5c38def8046f5a1a8589e2ec31711" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ad36199f9534c2db694d8d704696ff3" member_type: VOTER last_known_addr { host: "127.3.108.126" port: 33867 } } peers { permanent_uuid: "80dd9bfdcef247a9a343c37e9c1aa547" member_type: VOTER last_known_addr { host: "127.3.108.125" port: 39569 } } peers { permanent_uuid: "8fa5c38def8046f5a1a8589e2ec31711" member_type: VOTER last_known_addr { host: "127.3.108.124" port: 35887 } } }
15:07:12.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.535079 3750 sys_catalog.cc:458] T 00000000000000000000000000000000 P 80dd9bfdcef247a9a343c37e9c1aa547 [sys.catalog]: This master's current role is: FOLLOWER
15:07:12.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.532099 3783 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8ad36199f9534c2db694d8d704696ff3 [sys.catalog]: This master's current role is: FOLLOWER
15:07:12.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.536239 3783 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8ad36199f9534c2db694d8d704696ff3 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "8fa5c38def8046f5a1a8589e2ec31711" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ad36199f9534c2db694d8d704696ff3" member_type: VOTER last_known_addr { host: "127.3.108.126" port: 33867 } } peers { permanent_uuid: "80dd9bfdcef247a9a343c37e9c1aa547" member_type: VOTER last_known_addr { host: "127.3.108.125" port: 39569 } } peers { permanent_uuid: "8fa5c38def8046f5a1a8589e2ec31711" member_type: VOTER last_known_addr { host: "127.3.108.124" port: 35887 } } }
15:07:12.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.536455 3783 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8ad36199f9534c2db694d8d704696ff3 [sys.catalog]: This master's current role is: FOLLOWER
15:07:12.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.537024 3769 mvcc.cc:204] Tried to move back new op lower bound from 7113500395537989632 to 7113500395488546816. Current Snapshot: MvccSnapshot[applied={T|T < 7113500395537989632}]
15:07:12.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.537318 3784 mvcc.cc:204] Tried to move back new op lower bound from 7113500395537989632 to 7113500395488546816. Current Snapshot: MvccSnapshot[applied={T|T < 7113500395537989632}]
15:07:12.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.542690 3765 catalog_manager.cc:1348] Generated new cluster ID: 08803bde529741a6a033fcc296731b95
15:07:12.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.542774 3765 catalog_manager.cc:1496] Initializing Kudu internal certificate authority...
15:07:12.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.570570 3765 catalog_manager.cc:1371] Generated new certificate authority record
15:07:12.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.572098 3765 catalog_manager.cc:1505] Loading token signing keys...
15:07:12.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.587625 3765 catalog_manager.cc:5899] T 00000000000000000000000000000000 P 8fa5c38def8046f5a1a8589e2ec31711: Generated new TSK 0
15:07:12.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.588847 3765 catalog_manager.cc:1515] Initializing in-progress tserver states...
15:07:12.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.598008 3505 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskco6SfS/build/debug/bin/kudu
15:07:12.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskco6SfS/build/debug/bin/kudu
15:07:12.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/ts-0/wal
15:07:12.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/ts-0/data
15:07:12.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:07:12.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:07:12.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:07:12.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:07:12.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:07:12.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:07:12.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/ts-0/logs
15:07:12.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/ts-0/data/info.pb
15:07:12.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:07:12.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:07:12.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:07:12.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:07:12.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:07:12.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:07:12.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:07:12.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:07:12.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
15:07:12.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:07:12.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.108.65:0
15:07:12.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.108.65
15:07:12.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.108.65
15:07:12.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:07:12.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.108.126:33867,127.3.108.125:39569,127.3.108.124:35887
15:07:12.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.108.84:35745
15:07:12.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:07:12.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:07:12.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
15:07:12.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:12.744746 3790 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:07:12.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:12.744977 3790 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:07:12.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:12.745025 3790 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:07:12.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:12.748399 3790 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:07:12.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:12.748514 3790 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.3.108.65
15:07:12.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.752272 3790 tablet_server_runner.cc:78] Tablet server non-default flags:
15:07:12.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:07:12.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.108.84:35745
15:07:12.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:07:12.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:07:12.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/ts-0/data
15:07:12.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/ts-0/wal
15:07:12.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:07:12.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:07:12.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.108.65:0
15:07:12.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:07:12.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:07:12.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:07:12.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/ts-0/data/info.pb
15:07:12.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.108.65
15:07:12.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:07:12.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.108.126:33867,127.3.108.125:39569,127.3.108.124:35887
15:07:12.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:07:12.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.3790
15:07:12.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:07:12.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:07:12.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:07:12.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:07:12.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.108.65
15:07:12.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/ts-0/logs
15:07:12.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:07:12.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:07:12.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:07:12.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
15:07:12.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:07:12.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:07:12.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
15:07:12.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Jan 2025 14:43:14 UTC on 24a791456cd2
15:07:12.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4141
15:07:12.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.753350 3790 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:07:12.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.754418 3790 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:07:12.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:12.763381 3796 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:07:12.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:12.763746 3798 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:07:12.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:12.764050 3795 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:07:12.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.764897 3790 server_base.cc:1034] running on GCE node
15:07:12.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.765386 3790 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:07:12.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.766141 3790 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:07:12.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.767325 3790 hybrid_clock.cc:648] HybridClock initialized: now 1736694432767311 us; error 34 us; skew 500 ppm
15:07:12.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.769496 3790 webserver.cc:458] Webserver started at http://127.3.108.65:41295/ using document root <none> and password file <none>
15:07:12.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.770080 3790 fs_manager.cc:362] Metadata directory not provided
15:07:12.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.770140 3790 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:07:12.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.770350 3790 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:07:12.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.772045 3790 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/ts-0/data/instance:
15:07:12.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1a12be12d6cb436db08cb908e994066d"
15:07:12.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:07:12 on dist-test-slave-kjk3"
15:07:12.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.772569 3790 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/ts-0/wal/instance:
15:07:12.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1a12be12d6cb436db08cb908e994066d"
15:07:12.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:07:12 on dist-test-slave-kjk3"
15:07:12.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.776288 3790 fs_manager.cc:696] Time spent creating directory manager: real 0.003s user 0.004s sys 0.000s
15:07:12.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.778465 3804 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:07:12.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.779520 3790 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
15:07:12.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.779644 3790 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/ts-0/data,/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/ts-0/wal
15:07:12.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1a12be12d6cb436db08cb908e994066d"
15:07:12.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:07:12 on dist-test-slave-kjk3"
15:07:12.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.779748 3790 fs_report.cc:389] FS layout report
15:07:12.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:07:12.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/ts-0/wal
15:07:12.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/ts-0/wal
15:07:12.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/ts-0/data/data
15:07:12.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:07:12.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:07:12.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:07:12.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:07:12.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:07:12.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:07:12.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:07:12.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:07:12.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:07:12.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:07:12.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:07:12.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.794646 3790 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:07:12.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.795306 3790 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:07:12.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.795478 3790 kserver.cc:163] Server-wide thread pool size limit: 3276
15:07:12.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.796051 3790 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
15:07:12.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.797060 3790 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
15:07:12.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.797108 3790 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
15:07:12.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.797163 3790 ts_tablet_manager.cc:610] Registered 0 tablets
15:07:12.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.797181 3790 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
15:07:12.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.805866 3790 rpc_server.cc:307] RPC server started. Bound to: 127.3.108.65:37081
15:07:12.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.805923 3917 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.3.108.65:37081 every 8 connection(s)
15:07:12.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.806859 3790 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/ts-0/data/info.pb
15:07:12.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.808202 3505 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskco6SfS/build/debug/bin/kudu as pid 3790
15:07:12.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.808293 3505 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/ts-0/wal/instance
15:07:12.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.811422 3505 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskco6SfS/build/debug/bin/kudu
15:07:12.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskco6SfS/build/debug/bin/kudu
15:07:12.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/ts-1/wal
15:07:12.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/ts-1/data
15:07:12.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:07:12.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:07:12.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:07:12.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:07:12.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:07:12.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:07:12.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/ts-1/logs
15:07:12.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/ts-1/data/info.pb
15:07:12.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:07:12.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:07:12.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:07:12.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:07:12.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:07:12.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:07:12.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:07:12.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:07:12.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
15:07:12.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:07:12.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.108.66:0
15:07:12.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.108.66
15:07:12.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.108.66
15:07:12.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:07:12.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.108.126:33867,127.3.108.125:39569,127.3.108.124:35887
15:07:12.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.108.84:35745
15:07:12.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:07:12.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:07:12.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
15:07:12.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.818727 3918 heartbeater.cc:346] Connected to a master server at 127.3.108.124:35887
15:07:12.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.818738 3920 heartbeater.cc:346] Connected to a master server at 127.3.108.125:39569
15:07:12.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.818823 3919 heartbeater.cc:346] Connected to a master server at 127.3.108.126:33867
15:07:12.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.819005 3918 heartbeater.cc:463] Registering TS with master...
15:07:12.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.819005 3919 heartbeater.cc:463] Registering TS with master...
15:07:12.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.819005 3920 heartbeater.cc:463] Registering TS with master...
15:07:12.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.819720 3919 heartbeater.cc:510] Master 127.3.108.126:33867 requested a full tablet report, sending...
15:07:12.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.819720 3920 heartbeater.cc:510] Master 127.3.108.125:39569 requested a full tablet report, sending...
15:07:12.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.819721 3918 heartbeater.cc:510] Master 127.3.108.124:35887 requested a full tablet report, sending...
15:07:12.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.821328 3691 ts_manager.cc:194] Registered new tserver with Master: 1a12be12d6cb436db08cb908e994066d (127.3.108.65:37081)
15:07:12.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.821329 3613 ts_manager.cc:194] Registered new tserver with Master: 1a12be12d6cb436db08cb908e994066d (127.3.108.65:37081)
15:07:12.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.821384 3547 ts_manager.cc:194] Registered new tserver with Master: 1a12be12d6cb436db08cb908e994066d (127.3.108.65:37081)
15:07:12.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.822619 3691 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.3.108.65:59007
15:07:12.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:12.924068 3925 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:07:12.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:12.924353 3925 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:07:12.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:12.924401 3925 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:07:12.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:12.927628 3925 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:07:12.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:12.927754 3925 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.3.108.66
15:07:12.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.931730 3925 tablet_server_runner.cc:78] Tablet server non-default flags:
15:07:12.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:07:12.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.108.84:35745
15:07:12.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:07:12.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:07:12.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/ts-1/data
15:07:12.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/ts-1/wal
15:07:12.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:07:12.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:07:12.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.108.66:0
15:07:12.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:07:12.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:07:12.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:07:12.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/ts-1/data/info.pb
15:07:12.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.108.66
15:07:12.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:07:12.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.108.126:33867,127.3.108.125:39569,127.3.108.124:35887
15:07:12.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:07:12.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.3925
15:07:12.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:07:12.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:07:12.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:07:12.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:07:12.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.108.66
15:07:12.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/ts-1/logs
15:07:12.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:07:12.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:07:12.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:07:12.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
15:07:12.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:07:12.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:07:12.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
15:07:12.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Jan 2025 14:43:14 UTC on 24a791456cd2
15:07:12.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4141
15:07:12.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.932861 3925 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:07:12.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.933928 3925 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:07:12.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:12.942833 3930 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:07:12.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:12.943506 3931 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:07:12.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:12.943521 3933 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:07:12.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.944250 3925 server_base.cc:1034] running on GCE node
15:07:12.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.944718 3925 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:07:12.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.945356 3925 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:07:12.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.946532 3925 hybrid_clock.cc:648] HybridClock initialized: now 1736694432946504 us; error 48 us; skew 500 ppm
15:07:12.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.948637 3925 webserver.cc:458] Webserver started at http://127.3.108.66:39629/ using document root <none> and password file <none>
15:07:12.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.949184 3925 fs_manager.cc:362] Metadata directory not provided
15:07:12.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.949234 3925 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:07:12.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.949390 3925 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:07:12.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.951050 3925 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/ts-1/data/instance:
15:07:12.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e88a9cf6e6904c079287d933fd1421a9"
15:07:12.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:07:12 on dist-test-slave-kjk3"
15:07:12.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.951478 3925 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/ts-1/wal/instance:
15:07:12.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e88a9cf6e6904c079287d933fd1421a9"
15:07:12.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:07:12 on dist-test-slave-kjk3"
15:07:12.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.954980 3925 fs_manager.cc:696] Time spent creating directory manager: real 0.003s user 0.005s sys 0.000s
15:07:12.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.957355 3939 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:07:12.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.958590 3925 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.001s sys 0.000s
15:07:12.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.958706 3925 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/ts-1/data,/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/ts-1/wal
15:07:12.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e88a9cf6e6904c079287d933fd1421a9"
15:07:12.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:07:12 on dist-test-slave-kjk3"
15:07:12.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.958796 3925 fs_report.cc:389] FS layout report
15:07:12.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:07:12.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/ts-1/wal
15:07:12.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/ts-1/wal
15:07:12.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/ts-1/data/data
15:07:12.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:07:12.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:07:12.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:07:12.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:07:12.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:07:12.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:07:12.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:07:12.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:07:12.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:07:12.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:07:12.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:07:12.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.973656 3925 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:07:12.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.974256 3925 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:07:12.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.974408 3925 kserver.cc:163] Server-wide thread pool size limit: 3276
15:07:12.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.974965 3925 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
15:07:12.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.975874 3925 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
15:07:12.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.975924 3925 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
15:07:12.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.975976 3925 ts_tablet_manager.cc:610] Registered 0 tablets
15:07:12.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.975996 3925 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
15:07:12.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.984472 3925 rpc_server.cc:307] RPC server started. Bound to: 127.3.108.66:37025
15:07:12.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.984514 4052 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.3.108.66:37025 every 8 connection(s)
15:07:12.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.985436 3925 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/ts-1/data/info.pb
15:07:12.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.988679 3505 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskco6SfS/build/debug/bin/kudu as pid 3925
15:07:12.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.988782 3505 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/ts-1/wal/instance
15:07:12.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.991506 3505 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskco6SfS/build/debug/bin/kudu
15:07:12.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskco6SfS/build/debug/bin/kudu
15:07:12.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/ts-2/wal
15:07:12.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/ts-2/data
15:07:12.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:07:12.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:07:12.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:07:12.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:07:12.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:07:12.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:07:12.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/ts-2/logs
15:07:12.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/ts-2/data/info.pb
15:07:12.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:07:12.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:07:12.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:07:12.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:07:12.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:07:12.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:07:12.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:07:12.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:07:12.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
15:07:12.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:07:12.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.108.67:0
15:07:12.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.108.67
15:07:12.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.108.67
15:07:12.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:07:12.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.108.126:33867,127.3.108.125:39569,127.3.108.124:35887
15:07:12.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.108.84:35745
15:07:12.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:07:12.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:07:12.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
15:07:12.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.996628 4055 heartbeater.cc:346] Connected to a master server at 127.3.108.125:39569
15:07:12.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.996914 4055 heartbeater.cc:463] Registering TS with master...
15:07:12.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.997494 4055 heartbeater.cc:510] Master 127.3.108.125:39569 requested a full tablet report, sending...
15:07:12.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.996666 4054 heartbeater.cc:346] Connected to a master server at 127.3.108.126:33867
15:07:12.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.998034 4054 heartbeater.cc:463] Registering TS with master...
15:07:12.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.998253 4054 heartbeater.cc:510] Master 127.3.108.126:33867 requested a full tablet report, sending...
15:07:12.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.998880 3613 ts_manager.cc:194] Registered new tserver with Master: e88a9cf6e6904c079287d933fd1421a9 (127.3.108.66:37025)
15:07:12.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.999056 3547 ts_manager.cc:194] Registered new tserver with Master: e88a9cf6e6904c079287d933fd1421a9 (127.3.108.66:37025)
15:07:12.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.999171 4053 heartbeater.cc:346] Connected to a master server at 127.3.108.124:35887
15:07:12.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.999240 4053 heartbeater.cc:463] Registering TS with master...
15:07:12.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:12.999696 4053 heartbeater.cc:510] Master 127.3.108.124:35887 requested a full tablet report, sending...
15:07:13.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:13.000402 3691 ts_manager.cc:194] Registered new tserver with Master: e88a9cf6e6904c079287d933fd1421a9 (127.3.108.66:37025)
15:07:13.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:13.000957 3691 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.3.108.66:54933
15:07:13.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:13.113176 4060 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:07:13.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:13.113423 4060 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:07:13.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:13.113482 4060 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:07:13.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:13.117295 4060 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:07:13.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:13.117451 4060 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.3.108.67
15:07:13.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:13.121515 4060 tablet_server_runner.cc:78] Tablet server non-default flags:
15:07:13.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:07:13.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.108.84:35745
15:07:13.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:07:13.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:07:13.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/ts-2/data
15:07:13.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/ts-2/wal
15:07:13.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:07:13.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:07:13.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.108.67:0
15:07:13.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:07:13.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:07:13.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:07:13.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/ts-2/data/info.pb
15:07:13.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.108.67
15:07:13.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:07:13.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.108.126:33867,127.3.108.125:39569,127.3.108.124:35887
15:07:13.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:07:13.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.4060
15:07:13.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:07:13.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:07:13.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:07:13.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:07:13.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.108.67
15:07:13.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/ts-2/logs
15:07:13.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:07:13.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:07:13.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:07:13.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
15:07:13.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:07:13.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:07:13.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
15:07:13.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Jan 2025 14:43:14 UTC on 24a791456cd2
15:07:13.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4141
15:07:13.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:13.122826 4060 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:07:13.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:13.123968 4060 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:07:13.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:13.132017 4066 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:07:13.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:13.132943 4065 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:07:13.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:13.133409 4068 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:07:13.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:13.134254 4060 server_base.cc:1034] running on GCE node
15:07:13.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:13.134663 4060 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:07:13.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:13.135241 4060 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:07:13.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:13.136390 4060 hybrid_clock.cc:648] HybridClock initialized: now 1736694433136372 us; error 38 us; skew 500 ppm
15:07:13.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:13.138427 4060 webserver.cc:458] Webserver started at http://127.3.108.67:45879/ using document root <none> and password file <none>
15:07:13.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:13.138980 4060 fs_manager.cc:362] Metadata directory not provided
15:07:13.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:13.139030 4060 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:07:13.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:13.139173 4060 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:07:13.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:13.140890 4060 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/ts-2/data/instance:
15:07:13.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c6686df13e8846bcb64333968df098ab"
15:07:13.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:07:13 on dist-test-slave-kjk3"
15:07:13.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:13.141323 4060 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/ts-2/wal/instance:
15:07:13.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c6686df13e8846bcb64333968df098ab"
15:07:13.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:07:13 on dist-test-slave-kjk3"
15:07:13.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:13.144857 4060 fs_manager.cc:696] Time spent creating directory manager: real 0.003s user 0.002s sys 0.002s
15:07:13.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:13.147048 4074 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:07:13.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:13.148182 4060 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.003s sys 0.000s
15:07:13.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:13.148290 4060 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/ts-2/data,/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/ts-2/wal
15:07:13.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c6686df13e8846bcb64333968df098ab"
15:07:13.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:07:13 on dist-test-slave-kjk3"
15:07:13.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:13.148363 4060 fs_report.cc:389] FS layout report
15:07:13.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:07:13.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/ts-2/wal
15:07:13.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/ts-2/wal
15:07:13.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/ts-2/data/data
15:07:13.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:07:13.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:07:13.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:07:13.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:07:13.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:07:13.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:07:13.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:07:13.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:07:13.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:07:13.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:07:13.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:07:13.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:13.173199 4060 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:07:13.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:13.173791 4060 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:07:13.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:13.173930 4060 kserver.cc:163] Server-wide thread pool size limit: 3276
15:07:13.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:13.174458 4060 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
15:07:13.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:13.175357 4060 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
15:07:13.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:13.175401 4060 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
15:07:13.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:13.175438 4060 ts_tablet_manager.cc:610] Registered 0 tablets
15:07:13.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:13.175452 4060 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
15:07:13.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:13.183691 4060 rpc_server.cc:307] RPC server started. Bound to: 127.3.108.67:45159
15:07:13.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:13.183723 4187 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.3.108.67:45159 every 8 connection(s)
15:07:13.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:13.184774 4060 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/ts-2/data/info.pb
15:07:13.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:13.190091 3505 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskco6SfS/build/debug/bin/kudu as pid 4060
15:07:13.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:13.190202 3505 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster6374418845679082470/ts-2/wal/instance
15:07:13.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:13.194931 4189 heartbeater.cc:346] Connected to a master server at 127.3.108.126:33867
15:07:13.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:13.194996 4188 heartbeater.cc:346] Connected to a master server at 127.3.108.124:35887
15:07:13.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:13.195175 4189 heartbeater.cc:463] Registering TS with master...
15:07:13.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:13.195175 4188 heartbeater.cc:463] Registering TS with master...
15:07:13.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:13.195852 4188 heartbeater.cc:510] Master 127.3.108.124:35887 requested a full tablet report, sending...
15:07:13.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:13.195878 4190 heartbeater.cc:346] Connected to a master server at 127.3.108.125:39569
15:07:13.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:13.195852 4189 heartbeater.cc:510] Master 127.3.108.126:33867 requested a full tablet report, sending...
15:07:13.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:13.196024 4190 heartbeater.cc:463] Registering TS with master...
15:07:13.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:13.196223 4190 heartbeater.cc:510] Master 127.3.108.125:39569 requested a full tablet report, sending...
15:07:13.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:13.197036 3691 ts_manager.cc:194] Registered new tserver with Master: c6686df13e8846bcb64333968df098ab (127.3.108.67:45159)
15:07:13.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:13.197099 3615 ts_manager.cc:194] Registered new tserver with Master: c6686df13e8846bcb64333968df098ab (127.3.108.67:45159)
15:07:13.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:13.197099 3547 ts_manager.cc:194] Registered new tserver with Master: c6686df13e8846bcb64333968df098ab (127.3.108.67:45159)
15:07:13.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:13.197739 3691 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.3.108.67:51855
15:07:13.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:13.212433 3505 external_mini_cluster.cc:934] 3 TS(s) registered with all masters
15:07:13.213 [DEBUG - main] (MiniKuduCluster.java:195) Response:
15:07:13.213 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
15:07:13.214 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.3.108.126"
port: 33867
}
bound_http_address {
host: "127.3.108.126"
port: 33641
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.3.108.125"
port: 39569
}
bound_http_address {
host: "127.3.108.125"
port: 38841
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.3.108.124"
port: 35887
}
bound_http_address {
host: "127.3.108.124"
port: 34115
}
}
}
15:07:13.215 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
15:07:13.216 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.3.108.65"
port: 37081
}
bound_http_address {
host: "127.3.108.65"
port: 41295
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.3.108.66"
port: 37025
}
bound_http_address {
host: "127.3.108.66"
port: 39629
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.3.108.67"
port: 45159
}
bound_http_address {
host: "127.3.108.67"
port: 45879
}
}
}
15:07:13.217 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
15:07:13.217 [DEBUG - main] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster384470726150112034/krb5kdc/krb5cc
15:07:13.218 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
15:07:13.219 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskco6SfS/test-tmp
15:07:13.219 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/debug/bin
15:07:13.219 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskco6SfS/java/build/dist-test/../../../build/debug/bin/kudu, test, mini_cluster, --serialization=pb]
15:07:13.221 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
15:07:13.331 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:07:13.332 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
15:07:13.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:13.333094 4196 mini_oidc.cc:150] Starting JWKS server
15:07:13.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:13.334910 4196 webserver.cc:458] Webserver started at http://127.0.0.1:34843/ using document root <none> and password file <none>
15:07:13.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:13.335069 4196 mini_oidc.cc:186] Starting OIDC Discovery server
15:07:13.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:13.335304 4196 webserver.cc:458] Webserver started at http://127.0.0.1:43955/ using document root <none> and password file <none>
15:07:13.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
15:07:13.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/krb5kdc/principal' for realm 'KRBTEST.COM',
15:07:13.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
15:07:13.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 12 15:07:13 dist-test-slave-kjk3 krb5kdc[4206](info): setting up network...
15:07:13.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
15:07:13.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 12 15:07:13 dist-test-slave-kjk3 krb5kdc[4206](info): set up 2 sockets
15:07:13.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 12 15:07:13 dist-test-slave-kjk3 krb5kdc[4206](info): commencing operation
15:07:13.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
15:07:13.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:13.473920 3761 catalog_manager.cc:1260] Loaded cluster ID: 08803bde529741a6a033fcc296731b95
15:07:13.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:13.474094 3761 catalog_manager.cc:1553] T 00000000000000000000000000000000 P 80dd9bfdcef247a9a343c37e9c1aa547: loading cluster ID for follower catalog manager: success
15:07:13.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:13.476102 3761 catalog_manager.cc:1575] T 00000000000000000000000000000000 P 80dd9bfdcef247a9a343c37e9c1aa547: acquiring CA information for follower catalog manager: success
15:07:13.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:13.477573 3761 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 80dd9bfdcef247a9a343c37e9c1aa547: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:07:13.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:13.520854 3781 catalog_manager.cc:1260] Loaded cluster ID: 08803bde529741a6a033fcc296731b95
15:07:13.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:13.520941 3781 catalog_manager.cc:1553] T 00000000000000000000000000000000 P 8ad36199f9534c2db694d8d704696ff3: loading cluster ID for follower catalog manager: success
15:07:13.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:13.522786 3781 catalog_manager.cc:1575] T 00000000000000000000000000000000 P 8ad36199f9534c2db694d8d704696ff3: acquiring CA information for follower catalog manager: success
15:07:13.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:13.524006 3781 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 8ad36199f9534c2db694d8d704696ff3: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:07:13.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:13.825060 3918 heartbeater.cc:502] Master 127.3.108.124:35887 was elected leader, sending a full tablet report...
15:07:14.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.002748 4053 heartbeater.cc:502] Master 127.3.108.124:35887 was elected leader, sending a full tablet report...
15:07:14.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.199327 4188 heartbeater.cc:502] Master 127.3.108.124:35887 was elected leader, sending a full tablet report...
15:07:14.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:14.302445 4196 mini_kdc.cc:121] Time spent starting KDC: real 0.961s user 0.006s sys 0.000s
15:07:14.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
15:07:14.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
15:07:14.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
15:07:14.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
15:07:14.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
15:07:14.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
15:07:14.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
15:07:14.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
15:07:14.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
15:07:14.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
15:07:14.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/krb5kdc/test-user.keytab.
15:07:14.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/krb5kdc/test-user.keytab.
15:07:14.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 12 15:07:14 dist-test-slave-kjk3 krb5kdc[4206](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736694434, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:07:14.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
15:07:14.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-12T15:07:14Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
15:07:14.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-12T15:07:14Z Disabled control of system clock
15:07:14.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.4.25.62@KRBTEST.COM; defaulting to no policy
15:07:14.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:07:14.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.4.25.62@KRBTEST.COM" created.
15:07:14.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:07:14.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.4.25.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/krb5kdc/kudu_127.4.25.62.keytab.
15:07:14.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.4.25.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/krb5kdc/kudu_127.4.25.62.keytab.
15:07:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.419131 4196 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskco6SfS/build/debug/bin/kudu
15:07:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskco6SfS/build/debug/bin/kudu
15:07:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-0/wal
15:07:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-0/data
15:07:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:07:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:07:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:07:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:07:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:07:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:07:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-0/logs
15:07:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-0/data/info.pb
15:07:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:07:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:07:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:07:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:07:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:07:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:07:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:07:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:07:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:07:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:07:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:07:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:07:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.25.62:46827
15:07:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.25.62
15:07:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:07:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/krb5kdc/kudu_127.4.25.62.keytab
15:07:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.25.62
15:07:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:07:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:07:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:07:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.25.20:33809
15:07:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:07:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:07:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:07:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:07:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.25.62:46827,127.4.25.61:34743,127.4.25.60:41871
15:07:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:07:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34843/jwks.json
15:07:14.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:07:14.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:14.526257 4220 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:07:14.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:14.526511 4220 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:07:14.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:14.526589 4220 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:07:14.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:14.530028 4220 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:07:14.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:14.530108 4220 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:07:14.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:14.530144 4220 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:07:14.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:14.530169 4220 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:07:14.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:14.530189 4220 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:07:14.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:14.530213 4220 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:07:14.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:14.530229 4220 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34843/jwks.json
15:07:14.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.534950 4220 master_runner.cc:386] Master server non-default flags:
15:07:14.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:07:14.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.25.20:33809
15:07:14.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:07:14.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:07:14.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-0/data
15:07:14.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-0/wal
15:07:14.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:07:14.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:07:14.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.25.62:46827,127.4.25.61:34743,127.4.25.60:41871
15:07:14.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/krb5kdc/kudu_127.4.25.62.keytab
15:07:14.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.25.62
15:07:14.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:07:14.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:07:14.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:07:14.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.25.62:46827
15:07:14.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:07:14.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:07:14.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:07:14.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34843/jwks.json
15:07:14.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:07:14.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:07:14.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:07:14.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:07:14.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:07:14.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-0/data/info.pb
15:07:14.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.25.62
15:07:14.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:07:14.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:07:14.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.4220
15:07:14.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:07:14.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:07:14.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:07:14.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:07:14.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-0/logs
15:07:14.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:07:14.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:07:14.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:07:14.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:07:14.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:07:14.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:07:14.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
15:07:14.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Jan 2025 14:43:14 UTC on 24a791456cd2
15:07:14.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4141
15:07:14.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.536346 4220 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:07:14.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.537266 4220 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:07:14.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:14.545142 4226 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:07:14.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:14.545289 4225 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:07:14.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:14.546001 4228 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:07:14.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.546741 4220 server_base.cc:1034] running on GCE node
15:07:14.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.547283 4220 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:07:14.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.548381 4220 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:07:14.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.549554 4220 hybrid_clock.cc:648] HybridClock initialized: now 1736694434549534 us; error 33 us; skew 500 ppm
15:07:14.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 12 15:07:14 dist-test-slave-kjk3 krb5kdc[4206](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736694434, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.25.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:07:14.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.552373 4220 init.cc:377] Logged in from keytab as kudu/127.4.25.62@KRBTEST.COM (short username kudu)
15:07:14.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.553346 4220 webserver.cc:458] Webserver started at http://127.4.25.62:35181/ using document root <none> and password file <none>
15:07:14.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.553866 4220 fs_manager.cc:362] Metadata directory not provided
15:07:14.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.553917 4220 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:07:14.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.554061 4220 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:07:14.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.555682 4220 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-0/data/instance:
15:07:14.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c9c71ce8e09a4f17be3147a4299ff704"
15:07:14.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:07:14 on dist-test-slave-kjk3"
15:07:14.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.556166 4220 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-0/wal/instance:
15:07:14.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c9c71ce8e09a4f17be3147a4299ff704"
15:07:14.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:07:14 on dist-test-slave-kjk3"
15:07:14.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.559652 4220 fs_manager.cc:696] Time spent creating directory manager: real 0.003s user 0.002s sys 0.001s
15:07:14.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.561957 4235 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:07:14.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.563143 4220 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
15:07:14.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.563252 4220 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-0/data,/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-0/wal
15:07:14.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c9c71ce8e09a4f17be3147a4299ff704"
15:07:14.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:07:14 on dist-test-slave-kjk3"
15:07:14.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.563371 4220 fs_report.cc:389] FS layout report
15:07:14.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:07:14.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-0/wal
15:07:14.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-0/wal
15:07:14.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-0/data/data
15:07:14.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:07:14.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:07:14.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:07:14.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:07:14.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:07:14.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:07:14.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:07:14.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:07:14.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:07:14.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:07:14.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:07:14.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.595561 4220 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:07:14.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.598726 4220 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:07:14.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.598877 4220 kserver.cc:163] Server-wide thread pool size limit: 3276
15:07:14.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.605645 4220 rpc_server.cc:307] RPC server started. Bound to: 127.4.25.62:46827
15:07:14.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.605659 4287 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.4.25.62:46827 every 8 connection(s)
15:07:14.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.606674 4220 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-0/data/info.pb
15:07:14.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.609788 4288 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:07:14.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.613006 4288 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.25.62" port: 46827 } has no permanent_uuid. Determining permanent_uuid...
15:07:14.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.615000 4196 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskco6SfS/build/debug/bin/kudu as pid 4220
15:07:14.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.615137 4196 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-0/wal/instance
15:07:14.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 12 15:07:14 dist-test-slave-kjk3 krb5kdc[4206](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736694434, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.25.62@KRBTEST.COM for kudu/127.4.25.62@KRBTEST.COM
15:07:14.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.4.25.61@KRBTEST.COM; defaulting to no policy
15:07:14.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:07:14.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.4.25.61@KRBTEST.COM" created.
15:07:14.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.627808 4288 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.25.61" port: 34743 } has no permanent_uuid. Determining permanent_uuid...
15:07:14.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:14.628677 4237 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.25.61:34743: connect: Connection refused (error 111)
15:07:14.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:14.629575 4288 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.25.61:34743: Network error: Client connection negotiation failed: client connection to 127.4.25.61:34743: connect: Connection refused (error 111)
15:07:14.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:07:14.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.4.25.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/krb5kdc/kudu_127.4.25.61.keytab.
15:07:14.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.4.25.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/krb5kdc/kudu_127.4.25.61.keytab.
15:07:14.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.641500 4196 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskco6SfS/build/debug/bin/kudu
15:07:14.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskco6SfS/build/debug/bin/kudu
15:07:14.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-1/wal
15:07:14.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-1/data
15:07:14.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:07:14.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:07:14.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:07:14.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:07:14.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:07:14.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:07:14.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-1/logs
15:07:14.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-1/data/info.pb
15:07:14.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:07:14.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:07:14.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:07:14.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:07:14.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:07:14.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:07:14.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:07:14.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:07:14.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:07:14.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:07:14.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:07:14.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:07:14.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.25.61:34743
15:07:14.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.25.61
15:07:14.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:07:14.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/krb5kdc/kudu_127.4.25.61.keytab
15:07:14.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.25.61
15:07:14.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:07:14.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:07:14.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:07:14.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.25.20:33809
15:07:14.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:07:14.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:07:14.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:07:14.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:07:14.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.25.62:46827,127.4.25.61:34743,127.4.25.60:41871
15:07:14.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:07:14.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34843/jwks.json
15:07:14.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:07:14.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.678795 4288 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.25.61" port: 34743 } attempt: 1
15:07:14.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:14.680361 4288 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.25.61:34743: Network error: Client connection negotiation failed: client connection to 127.4.25.61:34743: connect: Connection refused (error 111)
15:07:14.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.748541 4288 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.25.61" port: 34743 } attempt: 2
15:07:14.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:14.749884 4288 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.25.61:34743: Network error: Client connection negotiation failed: client connection to 127.4.25.61:34743: connect: Connection refused (error 111)
15:07:14.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:14.752911 4295 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:07:14.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:14.753134 4295 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:07:14.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:14.753208 4295 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:07:14.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:14.756664 4295 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:07:14.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:14.756733 4295 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:07:14.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:14.756765 4295 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:07:14.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:14.756783 4295 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:07:14.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:14.756800 4295 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:07:14.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:14.756816 4295 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:07:14.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:14.756829 4295 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34843/jwks.json
15:07:14.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.761231 4295 master_runner.cc:386] Master server non-default flags:
15:07:14.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:07:14.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.25.20:33809
15:07:14.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:07:14.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:07:14.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-1/data
15:07:14.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-1/wal
15:07:14.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:07:14.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:07:14.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.25.62:46827,127.4.25.61:34743,127.4.25.60:41871
15:07:14.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/krb5kdc/kudu_127.4.25.61.keytab
15:07:14.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.25.61
15:07:14.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:07:14.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:07:14.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:07:14.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.25.61:34743
15:07:14.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:07:14.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:07:14.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:07:14.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34843/jwks.json
15:07:14.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:07:14.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:07:14.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:07:14.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:07:14.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:07:14.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-1/data/info.pb
15:07:14.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.25.61
15:07:14.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:07:14.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:07:14.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.4295
15:07:14.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:07:14.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:07:14.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:07:14.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:07:14.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-1/logs
15:07:14.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:07:14.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:07:14.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:07:14.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:07:14.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:07:14.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:07:14.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
15:07:14.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Jan 2025 14:43:14 UTC on 24a791456cd2
15:07:14.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4141
15:07:14.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.762253 4295 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:07:14.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.763118 4295 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:07:14.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:14.770597 4301 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:07:14.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:14.770778 4303 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:07:14.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:14.771322 4300 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:07:14.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.772250 4295 server_base.cc:1034] running on GCE node
15:07:14.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.772747 4295 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:07:14.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.773658 4295 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:07:14.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.774832 4295 hybrid_clock.cc:648] HybridClock initialized: now 1736694434774807 us; error 43 us; skew 500 ppm
15:07:14.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 12 15:07:14 dist-test-slave-kjk3 krb5kdc[4206](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736694434, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.25.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:07:14.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.777755 4295 init.cc:377] Logged in from keytab as kudu/127.4.25.61@KRBTEST.COM (short username kudu)
15:07:14.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.778774 4295 webserver.cc:458] Webserver started at http://127.4.25.61:43665/ using document root <none> and password file <none>
15:07:14.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.779337 4295 fs_manager.cc:362] Metadata directory not provided
15:07:14.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.779398 4295 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:07:14.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.779596 4295 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:07:14.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.781286 4295 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-1/data/instance:
15:07:14.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "edb0578f53e94642998053acdf451eaa"
15:07:14.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:07:14 on dist-test-slave-kjk3"
15:07:14.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.781734 4295 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-1/wal/instance:
15:07:14.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "edb0578f53e94642998053acdf451eaa"
15:07:14.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:07:14 on dist-test-slave-kjk3"
15:07:14.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.785043 4295 fs_manager.cc:696] Time spent creating directory manager: real 0.003s user 0.004s sys 0.000s
15:07:14.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.787112 4310 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:07:14.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.788215 4295 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.003s sys 0.000s
15:07:14.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.788333 4295 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-1/data,/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-1/wal
15:07:14.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "edb0578f53e94642998053acdf451eaa"
15:07:14.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:07:14 on dist-test-slave-kjk3"
15:07:14.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.788439 4295 fs_report.cc:389] FS layout report
15:07:14.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:07:14.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-1/wal
15:07:14.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-1/wal
15:07:14.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-1/data/data
15:07:14.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:07:14.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:07:14.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:07:14.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:07:14.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:07:14.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:07:14.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:07:14.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:07:14.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:07:14.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:07:14.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:07:14.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.803216 4295 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:07:14.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.805959 4295 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:07:14.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.806125 4295 kserver.cc:163] Server-wide thread pool size limit: 3276
15:07:14.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.812690 4295 rpc_server.cc:307] RPC server started. Bound to: 127.4.25.61:34743
15:07:14.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.812721 4362 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.4.25.61:34743 every 8 connection(s)
15:07:14.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.813730 4295 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-1/data/info.pb
15:07:14.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.816677 4196 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskco6SfS/build/debug/bin/kudu as pid 4295
15:07:14.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.816781 4196 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-1/wal/instance
15:07:14.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.816877 4363 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:07:14.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.820020 4363 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.25.62" port: 46827 } has no permanent_uuid. Determining permanent_uuid...
15:07:14.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.4.25.60@KRBTEST.COM; defaulting to no policy
15:07:14.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:07:14.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.4.25.60@KRBTEST.COM" created.
15:07:14.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 12 15:07:14 dist-test-slave-kjk3 krb5kdc[4206](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736694434, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.25.61@KRBTEST.COM for kudu/127.4.25.62@KRBTEST.COM
15:07:14.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.834314 4363 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.25.61" port: 34743 } has no permanent_uuid. Determining permanent_uuid...
15:07:14.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 12 15:07:14 dist-test-slave-kjk3 krb5kdc[4206](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736694434, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.25.61@KRBTEST.COM for kudu/127.4.25.61@KRBTEST.COM
15:07:14.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.841055 4288 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.25.61" port: 34743 } attempt: 3
15:07:14.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:07:14.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.4.25.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/krb5kdc/kudu_127.4.25.60.keytab.
15:07:14.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.4.25.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/krb5kdc/kudu_127.4.25.60.keytab.
15:07:14.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.844774 4196 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskco6SfS/build/debug/bin/kudu
15:07:14.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskco6SfS/build/debug/bin/kudu
15:07:14.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-2/wal
15:07:14.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-2/data
15:07:14.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:07:14.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:07:14.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:07:14.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:07:14.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:07:14.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:07:14.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-2/logs
15:07:14.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-2/data/info.pb
15:07:14.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:07:14.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:07:14.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:07:14.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:07:14.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:07:14.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:07:14.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:07:14.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:07:14.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:07:14.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:07:14.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:07:14.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:07:14.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.25.60:41871
15:07:14.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.25.60
15:07:14.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:07:14.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/krb5kdc/kudu_127.4.25.60.keytab
15:07:14.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.25.60
15:07:14.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:07:14.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:07:14.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:07:14.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.25.20:33809
15:07:14.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:07:14.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:07:14.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:07:14.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:07:14.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.25.62:46827,127.4.25.61:34743,127.4.25.60:41871
15:07:14.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:07:14.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34843/jwks.json
15:07:14.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:07:14.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.845088 4363 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.25.60" port: 41871 } has no permanent_uuid. Determining permanent_uuid...
15:07:14.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 12 15:07:14 dist-test-slave-kjk3 krb5kdc[4206](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736694434, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.25.62@KRBTEST.COM for kudu/127.4.25.61@KRBTEST.COM
15:07:14.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:14.846621 4314 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.25.60:41871: connect: Connection refused (error 111)
15:07:14.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:14.848481 4363 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.25.60:41871: Network error: Client connection negotiation failed: client connection to 127.4.25.60:41871: connect: Connection refused (error 111)
15:07:14.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.850842 4288 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.25.60" port: 41871 } has no permanent_uuid. Determining permanent_uuid...
15:07:14.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:14.852222 4288 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.25.60:41871: Network error: Client connection negotiation failed: client connection to 127.4.25.60:41871: connect: Connection refused (error 111)
15:07:14.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.883425 4288 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.25.60" port: 41871 } attempt: 1
15:07:14.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:14.885238 4288 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.25.60:41871: Network error: Client connection negotiation failed: client connection to 127.4.25.60:41871: connect: Connection refused (error 111)
15:07:14.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.897714 4363 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.25.60" port: 41871 } attempt: 1
15:07:14.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:14.899192 4363 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.25.60:41871: Network error: Client connection negotiation failed: client connection to 127.4.25.60:41871: connect: Connection refused (error 111)
15:07:14.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:14.957167 4370 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:07:14.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:14.957422 4370 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:07:14.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:14.957497 4370 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:07:14.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.960477 4288 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.25.60" port: 41871 } attempt: 2
15:07:14.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:14.960915 4370 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:07:14.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:14.960966 4370 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:07:14.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:14.961002 4370 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:07:14.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:14.961027 4370 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:07:14.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:14.961050 4370 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:07:14.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:14.961074 4370 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:07:14.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:14.961095 4370 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34843/jwks.json
15:07:14.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:14.961896 4288 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.25.60:41871: Network error: Client connection negotiation failed: client connection to 127.4.25.60:41871: connect: Connection refused (error 111)
15:07:14.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.965745 4370 master_runner.cc:386] Master server non-default flags:
15:07:14.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:07:14.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.25.20:33809
15:07:14.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:07:14.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:07:14.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-2/data
15:07:14.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-2/wal
15:07:14.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:07:14.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:07:14.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.25.62:46827,127.4.25.61:34743,127.4.25.60:41871
15:07:14.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/krb5kdc/kudu_127.4.25.60.keytab
15:07:14.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.25.60
15:07:14.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:07:14.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:07:14.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:07:14.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.25.60:41871
15:07:14.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:07:14.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:07:14.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:07:14.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34843/jwks.json
15:07:14.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:07:14.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:07:14.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:07:14.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:07:14.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:07:14.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-2/data/info.pb
15:07:14.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.25.60
15:07:14.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:07:14.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:07:14.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.4370
15:07:14.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:07:14.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:07:14.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:07:14.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:07:14.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-2/logs
15:07:14.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:07:14.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:07:14.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:07:14.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:07:14.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:07:14.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:07:14.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
15:07:14.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Jan 2025 14:43:14 UTC on 24a791456cd2
15:07:14.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4141
15:07:14.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.967510 4363 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.25.60" port: 41871 } attempt: 2
15:07:14.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.967862 4370 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:07:14.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.968819 4370 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:07:14.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:14.969065 4363 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.25.60:41871: Network error: Client connection negotiation failed: client connection to 127.4.25.60:41871: connect: Connection refused (error 111)
15:07:14.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:14.976184 4376 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:07:14.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:14.976642 4379 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:07:14.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:14.977524 4377 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:07:14.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.978127 4370 server_base.cc:1034] running on GCE node
15:07:14.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.978731 4370 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:07:14.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.979787 4370 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:07:14.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.980973 4370 hybrid_clock.cc:648] HybridClock initialized: now 1736694434980949 us; error 38 us; skew 500 ppm
15:07:14.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 12 15:07:14 dist-test-slave-kjk3 krb5kdc[4206](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736694434, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.25.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:07:14.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.983847 4370 init.cc:377] Logged in from keytab as kudu/127.4.25.60@KRBTEST.COM (short username kudu)
15:07:14.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.984899 4370 webserver.cc:458] Webserver started at http://127.4.25.60:33051/ using document root <none> and password file <none>
15:07:14.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.985487 4370 fs_manager.cc:362] Metadata directory not provided
15:07:14.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.985548 4370 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:07:14.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.985740 4370 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:07:14.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.987445 4370 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-2/data/instance:
15:07:14.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "83cb24a13df94c81ab682e2991154bb8"
15:07:14.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:07:14 on dist-test-slave-kjk3"
15:07:14.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.987938 4370 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-2/wal/instance:
15:07:14.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "83cb24a13df94c81ab682e2991154bb8"
15:07:14.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:07:14 on dist-test-slave-kjk3"
15:07:14.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.991503 4370 fs_manager.cc:696] Time spent creating directory manager: real 0.003s user 0.002s sys 0.001s
15:07:14.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.993959 4386 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:07:14.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.995127 4370 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
15:07:14.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.995249 4370 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-2/data,/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-2/wal
15:07:14.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "83cb24a13df94c81ab682e2991154bb8"
15:07:14.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:07:14 on dist-test-slave-kjk3"
15:07:14.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:14.995405 4370 fs_report.cc:389] FS layout report
15:07:14.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:07:14.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-2/wal
15:07:14.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-2/wal
15:07:14.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-2/data/data
15:07:14.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:07:14.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:07:14.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:07:14.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:07:14.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:07:14.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:07:14.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:07:14.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:07:14.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:07:14.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:07:14.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:07:15.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.012416 4370 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:07:15.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.015506 4370 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:07:15.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.015654 4370 kserver.cc:163] Server-wide thread pool size limit: 3276
15:07:15.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.021862 4370 rpc_server.cc:307] RPC server started. Bound to: 127.4.25.60:41871
15:07:15.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.021875 4438 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.4.25.60:41871 every 8 connection(s)
15:07:15.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.022846 4370 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-2/data/info.pb
15:07:15.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.025909 4439 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:07:15.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.029009 4439 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.25.62" port: 46827 } has no permanent_uuid. Determining permanent_uuid...
15:07:15.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.031484 4196 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskco6SfS/build/debug/bin/kudu as pid 4370
15:07:15.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.031595 4196 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-2/wal/instance
15:07:15.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 12 15:07:15 dist-test-slave-kjk3 krb5kdc[4206](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736694434, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.25.60@KRBTEST.COM for kudu/127.4.25.62@KRBTEST.COM
15:07:15.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.043622 4439 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.25.61" port: 34743 } has no permanent_uuid. Determining permanent_uuid...
15:07:15.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 12 15:07:15 dist-test-slave-kjk3 krb5kdc[4206](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736694434, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.4.25.60@KRBTEST.COM
15:07:15.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 12 15:07:15 dist-test-slave-kjk3 krb5kdc[4206](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736694434, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.25.60@KRBTEST.COM for kudu/127.4.25.61@KRBTEST.COM
15:07:15.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.051690 4439 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.25.60" port: 41871 } has no permanent_uuid. Determining permanent_uuid...
15:07:15.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 12 15:07:15 dist-test-slave-kjk3 krb5kdc[4206](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736694434, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.4.25.62@KRBTEST.COM
15:07:15.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 12 15:07:15 dist-test-slave-kjk3 krb5kdc[4206](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736694434, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.25.60@KRBTEST.COM for kudu/127.4.25.60@KRBTEST.COM
15:07:15.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.060252 4363 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.25.60" port: 41871 } attempt: 3
15:07:15.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.061197 4288 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.25.60" port: 41871 } attempt: 3
15:07:15.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.063371 4439 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 83cb24a13df94c81ab682e2991154bb8: Bootstrap starting.
15:07:15.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 12 15:07:15 dist-test-slave-kjk3 krb5kdc[4206](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736694434, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.25.62@KRBTEST.COM for kudu/127.4.25.60@KRBTEST.COM
15:07:15.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 12 15:07:15 dist-test-slave-kjk3 krb5kdc[4206](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736694434, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.25.61@KRBTEST.COM for kudu/127.4.25.60@KRBTEST.COM
15:07:15.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.066869 4439 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 83cb24a13df94c81ab682e2991154bb8: Neither blocks nor log segments found. Creating new log.
15:07:15.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 12 15:07:15 dist-test-slave-kjk3 krb5kdc[4206](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736694434, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.4.25.61@KRBTEST.COM
15:07:15.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.068029 4439 log.cc:826] T 00000000000000000000000000000000 P 83cb24a13df94c81ab682e2991154bb8: Log is configured to *not* fsync() on all Append() calls
15:07:15.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.070106 4439 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 83cb24a13df94c81ab682e2991154bb8: No bootstrap required, opened a new log
15:07:15.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.073571 4288 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704: Bootstrap starting.
15:07:15.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.073593 4363 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa: Bootstrap starting.
15:07:15.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.073483 4439 raft_consensus.cc:357] T 00000000000000000000000000000000 P 83cb24a13df94c81ab682e2991154bb8 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9c71ce8e09a4f17be3147a4299ff704" member_type: VOTER last_known_addr { host: "127.4.25.62" port: 46827 } } peers { permanent_uuid: "edb0578f53e94642998053acdf451eaa" member_type: VOTER last_known_addr { host: "127.4.25.61" port: 34743 } } peers { permanent_uuid: "83cb24a13df94c81ab682e2991154bb8" member_type: VOTER last_known_addr { host: "127.4.25.60" port: 41871 } }
15:07:15.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.073696 4439 raft_consensus.cc:383] T 00000000000000000000000000000000 P 83cb24a13df94c81ab682e2991154bb8 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:07:15.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.073742 4439 raft_consensus.cc:738] T 00000000000000000000000000000000 P 83cb24a13df94c81ab682e2991154bb8 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 83cb24a13df94c81ab682e2991154bb8, State: Initialized, Role: FOLLOWER
15:07:15.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.075032 4439 consensus_queue.cc:260] T 00000000000000000000000000000000 P 83cb24a13df94c81ab682e2991154bb8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9c71ce8e09a4f17be3147a4299ff704" member_type: VOTER last_known_addr { host: "127.4.25.62" port: 46827 } } peers { permanent_uuid: "edb0578f53e94642998053acdf451eaa" member_type: VOTER last_known_addr { host: "127.4.25.61" port: 34743 } } peers { permanent_uuid: "83cb24a13df94c81ab682e2991154bb8" member_type: VOTER last_known_addr { host: "127.4.25.60" port: 41871 } }
15:07:15.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.075565 4446 sys_catalog.cc:455] T 00000000000000000000000000000000 P 83cb24a13df94c81ab682e2991154bb8 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9c71ce8e09a4f17be3147a4299ff704" member_type: VOTER last_known_addr { host: "127.4.25.62" port: 46827 } } peers { permanent_uuid: "edb0578f53e94642998053acdf451eaa" member_type: VOTER last_known_addr { host: "127.4.25.61" port: 34743 } } peers { permanent_uuid: "83cb24a13df94c81ab682e2991154bb8" member_type: VOTER last_known_addr { host: "127.4.25.60" port: 41871 } } }
15:07:15.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.075814 4446 sys_catalog.cc:458] T 00000000000000000000000000000000 P 83cb24a13df94c81ab682e2991154bb8 [sys.catalog]: This master's current role is: FOLLOWER
15:07:15.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.075999 4288 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704: Neither blocks nor log segments found. Creating new log.
15:07:15.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.076000 4363 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa: Neither blocks nor log segments found. Creating new log.
15:07:15.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.076531 4439 sys_catalog.cc:564] T 00000000000000000000000000000000 P 83cb24a13df94c81ab682e2991154bb8 [sys.catalog]: configured and running, proceeding with master startup.
15:07:15.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.076798 4288 log.cc:826] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704: Log is configured to *not* fsync() on all Append() calls
15:07:15.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.076826 4363 log.cc:826] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa: Log is configured to *not* fsync() on all Append() calls
15:07:15.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.078617 4288 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704: No bootstrap required, opened a new log
15:07:15.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.080891 4196 external_mini_cluster.cc:934] 0 TS(s) registered with all masters
15:07:15.081 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:07:15.082 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
15:07:15.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:15.081979 4459 catalog_manager.cc:1559] T 00000000000000000000000000000000 P 83cb24a13df94c81ab682e2991154bb8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:07:15.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:15.082054 4459 catalog_manager.cc:874] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:07:15.082 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
15:07:15.083 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
15:07:15.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.080245 4370 master_runner.cc:186] Error getting master registration for 127.4.25.62:46827: OK, instance_id { permanent_uuid: "c9c71ce8e09a4f17be3147a4299ff704" instance_seqno: 1736694434603567 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
15:07:15.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.082724 4370 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:07:15.083 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.4.25.62"
port: 46827
}
bound_http_address {
host: "127.4.25.62"
port: 35181
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.4.25.61"
port: 34743
}
bound_http_address {
host: "127.4.25.61"
port: 43665
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.4.25.60"
port: 41871
}
bound_http_address {
host: "127.4.25.60"
port: 33051
}
}
}
15:07:15.084 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
15:07:15.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.085261 4288 raft_consensus.cc:357] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9c71ce8e09a4f17be3147a4299ff704" member_type: VOTER last_known_addr { host: "127.4.25.62" port: 46827 } } peers { permanent_uuid: "edb0578f53e94642998053acdf451eaa" member_type: VOTER last_known_addr { host: "127.4.25.61" port: 34743 } } peers { permanent_uuid: "83cb24a13df94c81ab682e2991154bb8" member_type: VOTER last_known_addr { host: "127.4.25.60" port: 41871 } }
15:07:15.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.085484 4288 raft_consensus.cc:383] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:07:15.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.085544 4288 raft_consensus.cc:738] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c9c71ce8e09a4f17be3147a4299ff704, State: Initialized, Role: FOLLOWER
15:07:15.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.086042 4288 consensus_queue.cc:260] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9c71ce8e09a4f17be3147a4299ff704" member_type: VOTER last_known_addr { host: "127.4.25.62" port: 46827 } } peers { permanent_uuid: "edb0578f53e94642998053acdf451eaa" member_type: VOTER last_known_addr { host: "127.4.25.61" port: 34743 } } peers { permanent_uuid: "83cb24a13df94c81ab682e2991154bb8" member_type: VOTER last_known_addr { host: "127.4.25.60" port: 41871 } }
15:07:15.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.086621 4461 sys_catalog.cc:455] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9c71ce8e09a4f17be3147a4299ff704" member_type: VOTER last_known_addr { host: "127.4.25.62" port: 46827 } } peers { permanent_uuid: "edb0578f53e94642998053acdf451eaa" member_type: VOTER last_known_addr { host: "127.4.25.61" port: 34743 } } peers { permanent_uuid: "83cb24a13df94c81ab682e2991154bb8" member_type: VOTER last_known_addr { host: "127.4.25.60" port: 41871 } } }
15:07:15.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.086760 4461 sys_catalog.cc:458] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704 [sys.catalog]: This master's current role is: FOLLOWER
15:07:15.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.087373 4288 sys_catalog.cc:564] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704 [sys.catalog]: configured and running, proceeding with master startup.
15:07:15.087 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:07:15.087 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
15:07:15.087 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
15:07:15.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.091468 4363 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa: No bootstrap required, opened a new log
15:07:15.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:15.093518 4473 catalog_manager.cc:1559] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:07:15.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:15.093794 4473 catalog_manager.cc:874] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:07:15.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.095104 4363 raft_consensus.cc:357] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9c71ce8e09a4f17be3147a4299ff704" member_type: VOTER last_known_addr { host: "127.4.25.62" port: 46827 } } peers { permanent_uuid: "edb0578f53e94642998053acdf451eaa" member_type: VOTER last_known_addr { host: "127.4.25.61" port: 34743 } } peers { permanent_uuid: "83cb24a13df94c81ab682e2991154bb8" member_type: VOTER last_known_addr { host: "127.4.25.60" port: 41871 } }
15:07:15.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.095319 4363 raft_consensus.cc:383] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:07:15.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.095386 4363 raft_consensus.cc:738] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: edb0578f53e94642998053acdf451eaa, State: Initialized, Role: FOLLOWER
15:07:15.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.096228 4363 consensus_queue.cc:260] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9c71ce8e09a4f17be3147a4299ff704" member_type: VOTER last_known_addr { host: "127.4.25.62" port: 46827 } } peers { permanent_uuid: "edb0578f53e94642998053acdf451eaa" member_type: VOTER last_known_addr { host: "127.4.25.61" port: 34743 } } peers { permanent_uuid: "83cb24a13df94c81ab682e2991154bb8" member_type: VOTER last_known_addr { host: "127.4.25.60" port: 41871 } }
15:07:15.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.097801 4363 sys_catalog.cc:564] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa [sys.catalog]: configured and running, proceeding with master startup.
15:07:15.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:15.100241 4241 tablet.cc:2367] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:07:15.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:15.103771 4486 catalog_manager.cc:1559] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:07:15.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:15.103849 4486 catalog_manager.cc:874] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:07:15.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.107030 4295 master_runner.cc:263] No leader master found from master edb0578f53e94642998053acdf451eaa
15:07:15.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.107100 4295 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:07:15.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.110829 4220 master_runner.cc:263] No leader master found from master c9c71ce8e09a4f17be3147a4299ff704
15:07:15.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.110924 4220 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:07:15.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.113610 4475 sys_catalog.cc:455] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9c71ce8e09a4f17be3147a4299ff704" member_type: VOTER last_known_addr { host: "127.4.25.62" port: 46827 } } peers { permanent_uuid: "edb0578f53e94642998053acdf451eaa" member_type: VOTER last_known_addr { host: "127.4.25.61" port: 34743 } } peers { permanent_uuid: "83cb24a13df94c81ab682e2991154bb8" member_type: VOTER last_known_addr { host: "127.4.25.60" port: 41871 } } }
15:07:15.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.113785 4475 sys_catalog.cc:458] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa [sys.catalog]: This master's current role is: FOLLOWER
15:07:15.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 12 15:07:15 dist-test-slave-kjk3 krb5kdc[4206](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736694435, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:07:15.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
15:07:15.133 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:07:15.134 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/krb5kdc/krb5cc
15:07:15.135 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
15:07:15.135 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/krb5kdc/krb5cc
15:07:15.135 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
15:07:15.137 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
15:07:15.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.156333 4461 raft_consensus.cc:491] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:07:15.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.156586 4461 raft_consensus.cc:513] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9c71ce8e09a4f17be3147a4299ff704" member_type: VOTER last_known_addr { host: "127.4.25.62" port: 46827 } } peers { permanent_uuid: "edb0578f53e94642998053acdf451eaa" member_type: VOTER last_known_addr { host: "127.4.25.61" port: 34743 } } peers { permanent_uuid: "83cb24a13df94c81ab682e2991154bb8" member_type: VOTER last_known_addr { host: "127.4.25.60" port: 41871 } }
15:07:15.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.157817 4461 leader_election.cc:290] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers edb0578f53e94642998053acdf451eaa (127.4.25.61:34743), 83cb24a13df94c81ab682e2991154bb8 (127.4.25.60:41871)
15:07:15.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.159447 4337 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c9c71ce8e09a4f17be3147a4299ff704" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "edb0578f53e94642998053acdf451eaa" is_pre_election: true
15:07:15.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.160580 4412 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c9c71ce8e09a4f17be3147a4299ff704" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "83cb24a13df94c81ab682e2991154bb8" is_pre_election: true
15:07:15.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.160905 4412 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 83cb24a13df94c81ab682e2991154bb8 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c9c71ce8e09a4f17be3147a4299ff704 in term 0.
15:07:15.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.161396 4239 leader_election.cc:304] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 83cb24a13df94c81ab682e2991154bb8, c9c71ce8e09a4f17be3147a4299ff704; no voters:
15:07:15.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.161742 4337 raft_consensus.cc:2463] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c9c71ce8e09a4f17be3147a4299ff704 in term 0.
15:07:15.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.162719 4461 raft_consensus.cc:2798] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704 [term 0 FOLLOWER]: Leader pre-election won for term 1
15:07:15.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.162802 4461 raft_consensus.cc:491] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
15:07:15.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.162842 4461 raft_consensus.cc:3054] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704 [term 0 FOLLOWER]: Advancing to term 1
15:07:15.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.163987 4461 raft_consensus.cc:513] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9c71ce8e09a4f17be3147a4299ff704" member_type: VOTER last_known_addr { host: "127.4.25.62" port: 46827 } } peers { permanent_uuid: "edb0578f53e94642998053acdf451eaa" member_type: VOTER last_known_addr { host: "127.4.25.61" port: 34743 } } peers { permanent_uuid: "83cb24a13df94c81ab682e2991154bb8" member_type: VOTER last_known_addr { host: "127.4.25.60" port: 41871 } }
15:07:15.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.164415 4461 leader_election.cc:290] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704 [CANDIDATE]: Term 1 election: Requested vote from peers edb0578f53e94642998053acdf451eaa (127.4.25.61:34743), 83cb24a13df94c81ab682e2991154bb8 (127.4.25.60:41871)
15:07:15.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.165148 4412 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c9c71ce8e09a4f17be3147a4299ff704" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "83cb24a13df94c81ab682e2991154bb8"
15:07:15.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.165273 4412 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 83cb24a13df94c81ab682e2991154bb8 [term 0 FOLLOWER]: Advancing to term 1
15:07:15.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.166399 4412 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 83cb24a13df94c81ab682e2991154bb8 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c9c71ce8e09a4f17be3147a4299ff704 in term 1.
15:07:15.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.166752 4239 leader_election.cc:304] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 83cb24a13df94c81ab682e2991154bb8, c9c71ce8e09a4f17be3147a4299ff704; no voters:
15:07:15.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.167433 4337 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c9c71ce8e09a4f17be3147a4299ff704" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "edb0578f53e94642998053acdf451eaa"
15:07:15.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.167577 4337 raft_consensus.cc:3054] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa [term 0 FOLLOWER]: Advancing to term 1
15:07:15.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.169564 4337 raft_consensus.cc:2463] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c9c71ce8e09a4f17be3147a4299ff704 in term 1.
15:07:15.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.170219 4461 raft_consensus.cc:2798] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704 [term 1 FOLLOWER]: Leader election won for term 1
15:07:15.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.170459 4461 raft_consensus.cc:695] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704 [term 1 LEADER]: Becoming Leader. State: Replica: c9c71ce8e09a4f17be3147a4299ff704, State: Running, Role: LEADER
15:07:15.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.170819 4461 consensus_queue.cc:237] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9c71ce8e09a4f17be3147a4299ff704" member_type: VOTER last_known_addr { host: "127.4.25.62" port: 46827 } } peers { permanent_uuid: "edb0578f53e94642998053acdf451eaa" member_type: VOTER last_known_addr { host: "127.4.25.61" port: 34743 } } peers { permanent_uuid: "83cb24a13df94c81ab682e2991154bb8" member_type: VOTER last_known_addr { host: "127.4.25.60" port: 41871 } }
15:07:15.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.172909 4461 sys_catalog.cc:455] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704 [sys.catalog]: SysCatalogTable state changed. Reason: New leader c9c71ce8e09a4f17be3147a4299ff704. Latest consensus state: current_term: 1 leader_uuid: "c9c71ce8e09a4f17be3147a4299ff704" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9c71ce8e09a4f17be3147a4299ff704" member_type: VOTER last_known_addr { host: "127.4.25.62" port: 46827 } } peers { permanent_uuid: "edb0578f53e94642998053acdf451eaa" member_type: VOTER last_known_addr { host: "127.4.25.61" port: 34743 } } peers { permanent_uuid: "83cb24a13df94c81ab682e2991154bb8" member_type: VOTER last_known_addr { host: "127.4.25.60" port: 41871 } } }
15:07:15.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.173209 4461 sys_catalog.cc:458] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704 [sys.catalog]: This master's current role is: LEADER
15:07:15.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.174191 4491 catalog_manager.cc:1476] Loading table and tablet metadata into memory...
15:07:15.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.176194 4491 catalog_manager.cc:1485] Initializing Kudu cluster ID...
15:07:15.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.184581 4412 raft_consensus.cc:1270] T 00000000000000000000000000000000 P 83cb24a13df94c81ab682e2991154bb8 [term 1 FOLLOWER]: Refusing update from remote peer c9c71ce8e09a4f17be3147a4299ff704: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
15:07:15.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.185591 4489 consensus_queue.cc:1035] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704 [LEADER]: Connected to new peer: Peer: permanent_uuid: "83cb24a13df94c81ab682e2991154bb8" member_type: VOTER last_known_addr { host: "127.4.25.60" port: 41871 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
15:07:15.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.196624 4446 sys_catalog.cc:455] T 00000000000000000000000000000000 P 83cb24a13df94c81ab682e2991154bb8 [sys.catalog]: SysCatalogTable state changed. Reason: New leader c9c71ce8e09a4f17be3147a4299ff704. Latest consensus state: current_term: 1 leader_uuid: "c9c71ce8e09a4f17be3147a4299ff704" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9c71ce8e09a4f17be3147a4299ff704" member_type: VOTER last_known_addr { host: "127.4.25.62" port: 46827 } } peers { permanent_uuid: "edb0578f53e94642998053acdf451eaa" member_type: VOTER last_known_addr { host: "127.4.25.61" port: 34743 } } peers { permanent_uuid: "83cb24a13df94c81ab682e2991154bb8" member_type: VOTER last_known_addr { host: "127.4.25.60" port: 41871 } } }
15:07:15.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.196792 4446 sys_catalog.cc:458] T 00000000000000000000000000000000 P 83cb24a13df94c81ab682e2991154bb8 [sys.catalog]: This master's current role is: FOLLOWER
15:07:15.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.198567 4337 raft_consensus.cc:1270] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa [term 1 FOLLOWER]: Refusing update from remote peer c9c71ce8e09a4f17be3147a4299ff704: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
15:07:15.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.200924 4489 consensus_queue.cc:1035] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704 [LEADER]: Connected to new peer: Peer: permanent_uuid: "edb0578f53e94642998053acdf451eaa" member_type: VOTER last_known_addr { host: "127.4.25.61" port: 34743 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
15:07:15.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.205629 4475 sys_catalog.cc:455] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa [sys.catalog]: SysCatalogTable state changed. Reason: New leader c9c71ce8e09a4f17be3147a4299ff704. Latest consensus state: current_term: 1 leader_uuid: "c9c71ce8e09a4f17be3147a4299ff704" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9c71ce8e09a4f17be3147a4299ff704" member_type: VOTER last_known_addr { host: "127.4.25.62" port: 46827 } } peers { permanent_uuid: "edb0578f53e94642998053acdf451eaa" member_type: VOTER last_known_addr { host: "127.4.25.61" port: 34743 } } peers { permanent_uuid: "83cb24a13df94c81ab682e2991154bb8" member_type: VOTER last_known_addr { host: "127.4.25.60" port: 41871 } } }
15:07:15.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.205790 4475 sys_catalog.cc:458] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa [sys.catalog]: This master's current role is: FOLLOWER
15:07:15.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.208425 4489 sys_catalog.cc:455] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "c9c71ce8e09a4f17be3147a4299ff704" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9c71ce8e09a4f17be3147a4299ff704" member_type: VOTER last_known_addr { host: "127.4.25.62" port: 46827 } } peers { permanent_uuid: "edb0578f53e94642998053acdf451eaa" member_type: VOTER last_known_addr { host: "127.4.25.61" port: 34743 } } peers { permanent_uuid: "83cb24a13df94c81ab682e2991154bb8" member_type: VOTER last_known_addr { host: "127.4.25.60" port: 41871 } } }
15:07:15.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.208566 4489 sys_catalog.cc:458] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704 [sys.catalog]: This master's current role is: LEADER
15:07:15.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.210094 4446 sys_catalog.cc:455] T 00000000000000000000000000000000 P 83cb24a13df94c81ab682e2991154bb8 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "c9c71ce8e09a4f17be3147a4299ff704" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9c71ce8e09a4f17be3147a4299ff704" member_type: VOTER last_known_addr { host: "127.4.25.62" port: 46827 } } peers { permanent_uuid: "edb0578f53e94642998053acdf451eaa" member_type: VOTER last_known_addr { host: "127.4.25.61" port: 34743 } } peers { permanent_uuid: "83cb24a13df94c81ab682e2991154bb8" member_type: VOTER last_known_addr { host: "127.4.25.60" port: 41871 } } }
15:07:15.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.210227 4446 sys_catalog.cc:458] T 00000000000000000000000000000000 P 83cb24a13df94c81ab682e2991154bb8 [sys.catalog]: This master's current role is: FOLLOWER
15:07:15.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.211997 4493 mvcc.cc:204] Tried to move back new op lower bound from 7113500406497607680 to 7113500406463991808. Current Snapshot: MvccSnapshot[applied={T|T < 7113500406497607680}]
15:07:15.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.212389 4492 mvcc.cc:204] Tried to move back new op lower bound from 7113500406497607680 to 7113500406463991808. Current Snapshot: MvccSnapshot[applied={T|T < 7113500406497607680}]
15:07:15.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.215726 4461 sys_catalog.cc:455] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "c9c71ce8e09a4f17be3147a4299ff704" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9c71ce8e09a4f17be3147a4299ff704" member_type: VOTER last_known_addr { host: "127.4.25.62" port: 46827 } } peers { permanent_uuid: "edb0578f53e94642998053acdf451eaa" member_type: VOTER last_known_addr { host: "127.4.25.61" port: 34743 } } peers { permanent_uuid: "83cb24a13df94c81ab682e2991154bb8" member_type: VOTER last_known_addr { host: "127.4.25.60" port: 41871 } } }
15:07:15.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.215857 4461 sys_catalog.cc:458] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704 [sys.catalog]: This master's current role is: LEADER
15:07:15.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.216490 4491 catalog_manager.cc:1348] Generated new cluster ID: d8ef055024ef43fb9fd0393c750b5864
15:07:15.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.216554 4491 catalog_manager.cc:1496] Initializing Kudu internal certificate authority...
15:07:15.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.217321 4475 sys_catalog.cc:455] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "c9c71ce8e09a4f17be3147a4299ff704" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9c71ce8e09a4f17be3147a4299ff704" member_type: VOTER last_known_addr { host: "127.4.25.62" port: 46827 } } peers { permanent_uuid: "edb0578f53e94642998053acdf451eaa" member_type: VOTER last_known_addr { host: "127.4.25.61" port: 34743 } } peers { permanent_uuid: "83cb24a13df94c81ab682e2991154bb8" member_type: VOTER last_known_addr { host: "127.4.25.60" port: 41871 } } }
15:07:15.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.217478 4475 sys_catalog.cc:458] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa [sys.catalog]: This master's current role is: FOLLOWER
15:07:15.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.218211 4495 mvcc.cc:204] Tried to move back new op lower bound from 7113500406497607680 to 7113500406463991808. Current Snapshot: MvccSnapshot[applied={T|T < 7113500406497607680}]
15:07:15.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.240244 4491 catalog_manager.cc:1371] Generated new certificate authority record
15:07:15.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.241506 4491 catalog_manager.cc:1505] Loading token signing keys...
15:07:15.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.260069 4491 catalog_manager.cc:5899] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704: Generated new TSK 0
15:07:15.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:15.261180 4491 catalog_manager.cc:1515] Initializing in-progress tserver states...
15:07:16.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:16.083246 4459 catalog_manager.cc:1260] Loaded cluster ID: d8ef055024ef43fb9fd0393c750b5864
15:07:16.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:16.083328 4459 catalog_manager.cc:1553] T 00000000000000000000000000000000 P 83cb24a13df94c81ab682e2991154bb8: loading cluster ID for follower catalog manager: success
15:07:16.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:16.085300 4459 catalog_manager.cc:1575] T 00000000000000000000000000000000 P 83cb24a13df94c81ab682e2991154bb8: acquiring CA information for follower catalog manager: success
15:07:16.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:16.086509 4459 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 83cb24a13df94c81ab682e2991154bb8: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:07:16.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:16.105101 4486 catalog_manager.cc:1260] Loaded cluster ID: d8ef055024ef43fb9fd0393c750b5864
15:07:16.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:16.105190 4486 catalog_manager.cc:1553] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa: loading cluster ID for follower catalog manager: success
15:07:16.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:16.106948 4486 catalog_manager.cc:1575] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa: acquiring CA information for follower catalog manager: success
15:07:16.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:16.108170 4486 catalog_manager.cc:1603] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:07:18.138 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.4.25.60:41871
15:07:18.138 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
15:07:18.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.139066 4196 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskco6SfS/build/debug/bin/kudu with pid 4370
15:07:18.144 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:07:18.144 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.4.25.62:46827
15:07:18.145 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
15:07:18.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.145607 4196 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskco6SfS/build/debug/bin/kudu with pid 4220
15:07:18.150 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:07:18.151 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.4.25.61:34743
15:07:18.151 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
15:07:18.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.151263 4196 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskco6SfS/build/debug/bin/kudu with pid 4295
15:07:18.156 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:07:18.156 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.4.25.60:41871
15:07:18.157 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
15:07:18.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.157433 4196 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskco6SfS/build/debug/bin/kudu
15:07:18.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskco6SfS/build/debug/bin/kudu
15:07:18.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-2/wal
15:07:18.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-2/data
15:07:18.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:07:18.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:07:18.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:07:18.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:07:18.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:07:18.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:07:18.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-2/logs
15:07:18.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-2/data/info.pb
15:07:18.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:07:18.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:07:18.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:07:18.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:07:18.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:07:18.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:07:18.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:07:18.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:07:18.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:07:18.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:07:18.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:07:18.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:07:18.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.25.60:41871
15:07:18.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.25.60
15:07:18.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33051
15:07:18.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/krb5kdc/kudu_127.4.25.60.keytab
15:07:18.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.25.60
15:07:18.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:07:18.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:07:18.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:07:18.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.25.20:33809
15:07:18.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:07:18.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:07:18.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:07:18.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:07:18.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.25.62:46827,127.4.25.61:34743,127.4.25.60:41871
15:07:18.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:07:18.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34843/jwks.json
15:07:18.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:07:18.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:18.268014 4501 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:07:18.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:18.268292 4501 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:07:18.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:18.268362 4501 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:07:18.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:18.271513 4501 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:07:18.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:18.271562 4501 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:07:18.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:18.271587 4501 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:07:18.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:18.271605 4501 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:07:18.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:18.271621 4501 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:07:18.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:18.271638 4501 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:07:18.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:18.271651 4501 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34843/jwks.json
15:07:18.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.276149 4501 master_runner.cc:386] Master server non-default flags:
15:07:18.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:07:18.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.25.20:33809
15:07:18.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:07:18.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:07:18.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-2/data
15:07:18.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-2/wal
15:07:18.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:07:18.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:07:18.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.25.62:46827,127.4.25.61:34743,127.4.25.60:41871
15:07:18.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/krb5kdc/kudu_127.4.25.60.keytab
15:07:18.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.25.60
15:07:18.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:07:18.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:07:18.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:07:18.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.25.60:41871
15:07:18.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:07:18.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:07:18.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:07:18.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34843/jwks.json
15:07:18.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:07:18.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:07:18.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:07:18.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:07:18.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:07:18.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-2/data/info.pb
15:07:18.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.25.60
15:07:18.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33051
15:07:18.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:07:18.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.4501
15:07:18.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:07:18.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:07:18.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:07:18.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:07:18.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-2/logs
15:07:18.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:07:18.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:07:18.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:07:18.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:07:18.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:07:18.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:07:18.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
15:07:18.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Jan 2025 14:43:14 UTC on 24a791456cd2
15:07:18.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4141
15:07:18.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.277254 4501 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:07:18.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.278138 4501 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:07:18.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:18.285181 4507 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:07:18.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:18.285982 4506 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:07:18.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:18.286520 4509 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:07:18.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.287400 4501 server_base.cc:1034] running on GCE node
15:07:18.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.287967 4501 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:07:18.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.289069 4501 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:07:18.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.290232 4501 hybrid_clock.cc:648] HybridClock initialized: now 1736694438290222 us; error 26 us; skew 500 ppm
15:07:18.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 12 15:07:18 dist-test-slave-kjk3 krb5kdc[4206](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736694438, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.25.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:07:18.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.296469 4501 init.cc:377] Logged in from keytab as kudu/127.4.25.60@KRBTEST.COM (short username kudu)
15:07:18.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.297461 4501 webserver.cc:458] Webserver started at http://127.4.25.60:33051/ using document root <none> and password file <none>
15:07:18.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.297988 4501 fs_manager.cc:362] Metadata directory not provided
15:07:18.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.298055 4501 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:07:18.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.301561 4501 fs_manager.cc:714] Time spent opening directory manager: real 0.002s user 0.003s sys 0.000s
15:07:18.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.303536 4516 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:07:18.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.304636 4501 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
15:07:18.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.304741 4501 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-2/data,/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-2/wal
15:07:18.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "83cb24a13df94c81ab682e2991154bb8"
15:07:18.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:07:14 on dist-test-slave-kjk3"
15:07:18.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.305141 4501 fs_report.cc:389] FS layout report
15:07:18.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:07:18.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-2/wal
15:07:18.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-2/wal
15:07:18.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-2/data/data
15:07:18.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:07:18.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:07:18.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:07:18.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:07:18.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:07:18.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:07:18.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:07:18.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:07:18.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:07:18.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:07:18.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:07:18.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.326143 4501 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:07:18.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.329151 4501 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:07:18.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.329305 4501 kserver.cc:163] Server-wide thread pool size limit: 3276
15:07:18.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.336009 4568 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.4.25.60:41871 every 8 connection(s)
15:07:18.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.336009 4501 rpc_server.cc:307] RPC server started. Bound to: 127.4.25.60:41871
15:07:18.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.337210 4501 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-2/data/info.pb
15:07:18.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.341346 4569 sys_catalog.cc:263] Verifying existing consensus state
15:07:18.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.343055 4196 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskco6SfS/build/debug/bin/kudu as pid 4501
15:07:18.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.343071 4569 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 83cb24a13df94c81ab682e2991154bb8: Bootstrap starting.
15:07:18.343 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:07:18.343 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.4.25.62:46827
15:07:18.343 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
15:07:18.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.343994 4196 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskco6SfS/build/debug/bin/kudu
15:07:18.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskco6SfS/build/debug/bin/kudu
15:07:18.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-0/wal
15:07:18.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-0/data
15:07:18.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:07:18.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:07:18.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:07:18.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:07:18.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:07:18.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:07:18.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-0/logs
15:07:18.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-0/data/info.pb
15:07:18.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:07:18.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:07:18.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:07:18.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:07:18.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:07:18.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:07:18.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:07:18.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:07:18.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:07:18.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:07:18.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:07:18.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:07:18.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.25.62:46827
15:07:18.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.25.62
15:07:18.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35181
15:07:18.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/krb5kdc/kudu_127.4.25.62.keytab
15:07:18.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.25.62
15:07:18.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:07:18.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:07:18.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:07:18.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.25.20:33809
15:07:18.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:07:18.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:07:18.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:07:18.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:07:18.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.25.62:46827,127.4.25.61:34743,127.4.25.60:41871
15:07:18.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:07:18.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34843/jwks.json
15:07:18.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:07:18.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.357594 4569 log.cc:826] T 00000000000000000000000000000000 P 83cb24a13df94c81ab682e2991154bb8: Log is configured to *not* fsync() on all Append() calls
15:07:18.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.363909 4569 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 83cb24a13df94c81ab682e2991154bb8: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
15:07:18.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.364310 4569 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 83cb24a13df94c81ab682e2991154bb8: Bootstrap complete.
15:07:18.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.367671 4569 raft_consensus.cc:357] T 00000000000000000000000000000000 P 83cb24a13df94c81ab682e2991154bb8 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9c71ce8e09a4f17be3147a4299ff704" member_type: VOTER last_known_addr { host: "127.4.25.62" port: 46827 } } peers { permanent_uuid: "edb0578f53e94642998053acdf451eaa" member_type: VOTER last_known_addr { host: "127.4.25.61" port: 34743 } } peers { permanent_uuid: "83cb24a13df94c81ab682e2991154bb8" member_type: VOTER last_known_addr { host: "127.4.25.60" port: 41871 } }
15:07:18.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.368211 4569 raft_consensus.cc:738] T 00000000000000000000000000000000 P 83cb24a13df94c81ab682e2991154bb8 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 83cb24a13df94c81ab682e2991154bb8, State: Initialized, Role: FOLLOWER
15:07:18.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.368726 4569 consensus_queue.cc:260] T 00000000000000000000000000000000 P 83cb24a13df94c81ab682e2991154bb8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9c71ce8e09a4f17be3147a4299ff704" member_type: VOTER last_known_addr { host: "127.4.25.62" port: 46827 } } peers { permanent_uuid: "edb0578f53e94642998053acdf451eaa" member_type: VOTER last_known_addr { host: "127.4.25.61" port: 34743 } } peers { permanent_uuid: "83cb24a13df94c81ab682e2991154bb8" member_type: VOTER last_known_addr { host: "127.4.25.60" port: 41871 } }
15:07:18.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.369180 4573 sys_catalog.cc:455] T 00000000000000000000000000000000 P 83cb24a13df94c81ab682e2991154bb8 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9c71ce8e09a4f17be3147a4299ff704" member_type: VOTER last_known_addr { host: "127.4.25.62" port: 46827 } } peers { permanent_uuid: "edb0578f53e94642998053acdf451eaa" member_type: VOTER last_known_addr { host: "127.4.25.61" port: 34743 } } peers { permanent_uuid: "83cb24a13df94c81ab682e2991154bb8" member_type: VOTER last_known_addr { host: "127.4.25.60" port: 41871 } } }
15:07:18.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.369331 4573 sys_catalog.cc:458] T 00000000000000000000000000000000 P 83cb24a13df94c81ab682e2991154bb8 [sys.catalog]: This master's current role is: FOLLOWER
15:07:18.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.369879 4569 sys_catalog.cc:564] T 00000000000000000000000000000000 P 83cb24a13df94c81ab682e2991154bb8 [sys.catalog]: configured and running, proceeding with master startup.
15:07:18.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.374735 4584 catalog_manager.cc:1260] Loaded cluster ID: d8ef055024ef43fb9fd0393c750b5864
15:07:18.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.374804 4584 catalog_manager.cc:1553] T 00000000000000000000000000000000 P 83cb24a13df94c81ab682e2991154bb8: loading cluster ID for follower catalog manager: success
15:07:18.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.377159 4584 catalog_manager.cc:1575] T 00000000000000000000000000000000 P 83cb24a13df94c81ab682e2991154bb8: acquiring CA information for follower catalog manager: success
15:07:18.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.378383 4584 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 83cb24a13df94c81ab682e2991154bb8: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:07:18.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.376780 4501 master_runner.cc:186] Error getting master registration for 127.4.25.62:46827: Network error: Client connection negotiation failed: client connection to 127.4.25.62:46827: connect: Connection refused (error 111),
15:07:18.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.379015 4501 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:07:18.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:18.454562 4571 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:07:18.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:18.454792 4571 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:07:18.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:18.454847 4571 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:07:18.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:18.458057 4571 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:07:18.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:18.458115 4571 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:07:18.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:18.458141 4571 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:07:18.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:18.458177 4571 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:07:18.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:18.458195 4571 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:07:18.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:18.458211 4571 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:07:18.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:18.458225 4571 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34843/jwks.json
15:07:18.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.462628 4571 master_runner.cc:386] Master server non-default flags:
15:07:18.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:07:18.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.25.20:33809
15:07:18.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:07:18.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:07:18.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-0/data
15:07:18.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-0/wal
15:07:18.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:07:18.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:07:18.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.25.62:46827,127.4.25.61:34743,127.4.25.60:41871
15:07:18.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/krb5kdc/kudu_127.4.25.62.keytab
15:07:18.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.25.62
15:07:18.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:07:18.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:07:18.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:07:18.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.25.62:46827
15:07:18.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:07:18.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:07:18.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:07:18.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34843/jwks.json
15:07:18.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:07:18.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:07:18.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:07:18.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:07:18.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:07:18.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-0/data/info.pb
15:07:18.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.25.62
15:07:18.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35181
15:07:18.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:07:18.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.4571
15:07:18.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:07:18.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:07:18.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:07:18.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:07:18.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-0/logs
15:07:18.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:07:18.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:07:18.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:07:18.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:07:18.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:07:18.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:07:18.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
15:07:18.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Jan 2025 14:43:14 UTC on 24a791456cd2
15:07:18.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4141
15:07:18.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.463707 4571 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:07:18.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.464529 4571 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:07:18.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:18.472528 4592 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:07:18.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:18.473183 4594 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:07:18.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:18.473419 4591 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:07:18.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.474045 4571 server_base.cc:1034] running on GCE node
15:07:18.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.474649 4571 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:07:18.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.475788 4571 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:07:18.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.476979 4571 hybrid_clock.cc:648] HybridClock initialized: now 1736694438476954 us; error 40 us; skew 500 ppm
15:07:18.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 12 15:07:18 dist-test-slave-kjk3 krb5kdc[4206](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736694438, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.25.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:07:18.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.480237 4571 init.cc:377] Logged in from keytab as kudu/127.4.25.62@KRBTEST.COM (short username kudu)
15:07:18.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.481348 4571 webserver.cc:458] Webserver started at http://127.4.25.62:35181/ using document root <none> and password file <none>
15:07:18.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.482000 4571 fs_manager.cc:362] Metadata directory not provided
15:07:18.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.482084 4571 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:07:18.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.486426 4571 fs_manager.cc:714] Time spent opening directory manager: real 0.003s user 0.003s sys 0.000s
15:07:18.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.488772 4601 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:07:18.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.489921 4571 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.003s sys 0.000s
15:07:18.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.490039 4571 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-0/data,/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-0/wal
15:07:18.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c9c71ce8e09a4f17be3147a4299ff704"
15:07:18.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:07:14 on dist-test-slave-kjk3"
15:07:18.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.490564 4571 fs_report.cc:389] FS layout report
15:07:18.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:07:18.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-0/wal
15:07:18.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-0/wal
15:07:18.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-0/data/data
15:07:18.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:07:18.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:07:18.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:07:18.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:07:18.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:07:18.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:07:18.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:07:18.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:07:18.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:07:18.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:07:18.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:07:18.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.509408 4571 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:07:18.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.512914 4571 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:07:18.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.513109 4571 kserver.cc:163] Server-wide thread pool size limit: 3276
15:07:18.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.521790 4653 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.4.25.62:46827 every 8 connection(s)
15:07:18.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.521808 4571 rpc_server.cc:307] RPC server started. Bound to: 127.4.25.62:46827
15:07:18.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.523244 4571 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-0/data/info.pb
15:07:18.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.527496 4654 sys_catalog.cc:263] Verifying existing consensus state
15:07:18.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.529242 4654 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704: Bootstrap starting.
15:07:18.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.531019 4196 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskco6SfS/build/debug/bin/kudu as pid 4571
15:07:18.531 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:07:18.531 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.4.25.61:34743
15:07:18.531 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
15:07:18.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.532097 4196 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskco6SfS/build/debug/bin/kudu
15:07:18.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskco6SfS/build/debug/bin/kudu
15:07:18.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-1/wal
15:07:18.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-1/data
15:07:18.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:07:18.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:07:18.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:07:18.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:07:18.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:07:18.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:07:18.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-1/logs
15:07:18.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-1/data/info.pb
15:07:18.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:07:18.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:07:18.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:07:18.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:07:18.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:07:18.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:07:18.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:07:18.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:07:18.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:07:18.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:07:18.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:07:18.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:07:18.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.25.61:34743
15:07:18.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.25.61
15:07:18.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43665
15:07:18.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/krb5kdc/kudu_127.4.25.61.keytab
15:07:18.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.25.61
15:07:18.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:07:18.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:07:18.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:07:18.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.25.20:33809
15:07:18.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:07:18.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:07:18.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:07:18.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:07:18.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.25.62:46827,127.4.25.61:34743,127.4.25.60:41871
15:07:18.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:07:18.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34843/jwks.json
15:07:18.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:07:18.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.545316 4654 log.cc:826] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704: Log is configured to *not* fsync() on all Append() calls
15:07:18.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.552472 4654 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
15:07:18.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.552874 4654 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704: Bootstrap complete.
15:07:18.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.557220 4654 raft_consensus.cc:357] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9c71ce8e09a4f17be3147a4299ff704" member_type: VOTER last_known_addr { host: "127.4.25.62" port: 46827 } } peers { permanent_uuid: "edb0578f53e94642998053acdf451eaa" member_type: VOTER last_known_addr { host: "127.4.25.61" port: 34743 } } peers { permanent_uuid: "83cb24a13df94c81ab682e2991154bb8" member_type: VOTER last_known_addr { host: "127.4.25.60" port: 41871 } }
15:07:18.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.557624 4654 raft_consensus.cc:738] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: c9c71ce8e09a4f17be3147a4299ff704, State: Initialized, Role: FOLLOWER
15:07:18.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.558130 4654 consensus_queue.cc:260] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9c71ce8e09a4f17be3147a4299ff704" member_type: VOTER last_known_addr { host: "127.4.25.62" port: 46827 } } peers { permanent_uuid: "edb0578f53e94642998053acdf451eaa" member_type: VOTER last_known_addr { host: "127.4.25.61" port: 34743 } } peers { permanent_uuid: "83cb24a13df94c81ab682e2991154bb8" member_type: VOTER last_known_addr { host: "127.4.25.60" port: 41871 } }
15:07:18.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.558600 4658 sys_catalog.cc:455] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9c71ce8e09a4f17be3147a4299ff704" member_type: VOTER last_known_addr { host: "127.4.25.62" port: 46827 } } peers { permanent_uuid: "edb0578f53e94642998053acdf451eaa" member_type: VOTER last_known_addr { host: "127.4.25.61" port: 34743 } } peers { permanent_uuid: "83cb24a13df94c81ab682e2991154bb8" member_type: VOTER last_known_addr { host: "127.4.25.60" port: 41871 } } }
15:07:18.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.558717 4658 sys_catalog.cc:458] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704 [sys.catalog]: This master's current role is: FOLLOWER
15:07:18.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.559280 4654 sys_catalog.cc:564] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704 [sys.catalog]: configured and running, proceeding with master startup.
15:07:18.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.565294 4669 catalog_manager.cc:1260] Loaded cluster ID: d8ef055024ef43fb9fd0393c750b5864
15:07:18.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.565373 4669 catalog_manager.cc:1553] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704: loading cluster ID for follower catalog manager: success
15:07:18.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.568148 4669 catalog_manager.cc:1575] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704: acquiring CA information for follower catalog manager: success
15:07:18.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.569818 4669 catalog_manager.cc:1603] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:07:18.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 12 15:07:18 dist-test-slave-kjk3 krb5kdc[4206](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736694438, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.25.62@KRBTEST.COM for kudu/127.4.25.62@KRBTEST.COM
15:07:18.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.579221 4571 master_runner.cc:186] Error getting master registration for 127.4.25.61:34743: Network error: Client connection negotiation failed: client connection to 127.4.25.61:34743: connect: Connection refused (error 111),
15:07:18.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.581519 4571 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:07:18.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:18.651683 4656 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:07:18.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:18.651912 4656 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:07:18.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:18.651984 4656 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:07:18.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:18.655193 4656 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:07:18.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:18.655249 4656 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:07:18.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:18.655284 4656 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:07:18.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:18.655306 4656 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:07:18.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:18.655328 4656 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:07:18.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:18.655350 4656 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:07:18.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:18.655368 4656 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34843/jwks.json
15:07:18.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.659847 4656 master_runner.cc:386] Master server non-default flags:
15:07:18.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:07:18.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.25.20:33809
15:07:18.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:07:18.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:07:18.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-1/data
15:07:18.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-1/wal
15:07:18.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:07:18.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:07:18.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.25.62:46827,127.4.25.61:34743,127.4.25.60:41871
15:07:18.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/krb5kdc/kudu_127.4.25.61.keytab
15:07:18.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.25.61
15:07:18.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:07:18.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:07:18.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:07:18.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.25.61:34743
15:07:18.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:07:18.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:07:18.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:07:18.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34843/jwks.json
15:07:18.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:07:18.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:07:18.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:07:18.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:07:18.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:07:18.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-1/data/info.pb
15:07:18.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.25.61
15:07:18.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43665
15:07:18.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:07:18.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.4656
15:07:18.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:07:18.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:07:18.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:07:18.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:07:18.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-1/logs
15:07:18.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:07:18.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:07:18.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:07:18.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:07:18.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:07:18.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:07:18.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
15:07:18.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Jan 2025 14:43:14 UTC on 24a791456cd2
15:07:18.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4141
15:07:18.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.661099 4656 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:07:18.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.661931 4656 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:07:18.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:18.668905 4678 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:07:18.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:18.669485 4677 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:07:18.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:18.669986 4680 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:07:18.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.670842 4656 server_base.cc:1034] running on GCE node
15:07:18.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.671407 4656 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:07:18.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.672626 4656 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:07:18.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.673810 4656 hybrid_clock.cc:648] HybridClock initialized: now 1736694438673791 us; error 34 us; skew 500 ppm
15:07:18.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 12 15:07:18 dist-test-slave-kjk3 krb5kdc[4206](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736694438, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.25.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:07:18.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.676985 4656 init.cc:377] Logged in from keytab as kudu/127.4.25.61@KRBTEST.COM (short username kudu)
15:07:18.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.678134 4656 webserver.cc:458] Webserver started at http://127.4.25.61:43665/ using document root <none> and password file <none>
15:07:18.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.678910 4656 fs_manager.cc:362] Metadata directory not provided
15:07:18.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.678997 4656 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:07:18.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.683220 4656 fs_manager.cc:714] Time spent opening directory manager: real 0.003s user 0.002s sys 0.000s
15:07:18.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.685550 4687 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:07:18.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.686699 4656 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.001s sys 0.000s
15:07:18.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.686820 4656 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-1/data,/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-1/wal
15:07:18.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "edb0578f53e94642998053acdf451eaa"
15:07:18.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:07:14 on dist-test-slave-kjk3"
15:07:18.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.687317 4656 fs_report.cc:389] FS layout report
15:07:18.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:07:18.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-1/wal
15:07:18.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-1/wal
15:07:18.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-1/data/data
15:07:18.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:07:18.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:07:18.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:07:18.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:07:18.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:07:18.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:07:18.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:07:18.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:07:18.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:07:18.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:07:18.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:07:18.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.713647 4656 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:07:18.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.717406 4656 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:07:18.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.717584 4656 kserver.cc:163] Server-wide thread pool size limit: 3276
15:07:18.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.725420 4739 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.4.25.61:34743 every 8 connection(s)
15:07:18.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.725427 4656 rpc_server.cc:307] RPC server started. Bound to: 127.4.25.61:34743
15:07:18.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.726742 4656 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-1/data/info.pb
15:07:18.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.729179 4196 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskco6SfS/build/debug/bin/kudu as pid 4656
15:07:18.729 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:07:18.730 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.4.25.60:41871(127.4.25.60:41871)] connecting to peer
15:07:18.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.731101 4740 sys_catalog.cc:263] Verifying existing consensus state
15:07:18.731 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.4.25.62:46827(127.4.25.62:46827)] connecting to peer
15:07:18.732 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.4.25.61:34743(127.4.25.61:34743)] connecting to peer
15:07:18.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.733245 4740 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa: Bootstrap starting.
15:07:18.733 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.4.25.62:46827(127.4.25.62:46827)] Successfully connected to peer
15:07:18.734 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.4.25.60:41871(127.4.25.60:41871)] Successfully connected to peer
15:07:18.735 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.4.25.61:34743(127.4.25.61:34743)] Successfully connected to peer
15:07:18.738 [WARN - kudu-nio-1] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
15:07:18.739 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.4.25.61
15:07:18.740 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.4.25.62
15:07:18.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.750631 4740 log.cc:826] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa: Log is configured to *not* fsync() on all Append() calls
15:07:18.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 12 15:07:18 dist-test-slave-kjk3 krb5kdc[4206](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736694435, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.4.25.62@KRBTEST.COM
15:07:18.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 12 15:07:18 dist-test-slave-kjk3 krb5kdc[4206](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736694435, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.4.25.61@KRBTEST.COM
15:07:18.758 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.4.25.60
15:07:18.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.758960 4740 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
15:07:18.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.759457 4740 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa: Bootstrap complete.
15:07:18.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.763789 4740 raft_consensus.cc:357] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9c71ce8e09a4f17be3147a4299ff704" member_type: VOTER last_known_addr { host: "127.4.25.62" port: 46827 } } peers { permanent_uuid: "edb0578f53e94642998053acdf451eaa" member_type: VOTER last_known_addr { host: "127.4.25.61" port: 34743 } } peers { permanent_uuid: "83cb24a13df94c81ab682e2991154bb8" member_type: VOTER last_known_addr { host: "127.4.25.60" port: 41871 } }
15:07:18.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.764285 4740 raft_consensus.cc:738] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: edb0578f53e94642998053acdf451eaa, State: Initialized, Role: FOLLOWER
15:07:18.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.764827 4740 consensus_queue.cc:260] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9c71ce8e09a4f17be3147a4299ff704" member_type: VOTER last_known_addr { host: "127.4.25.62" port: 46827 } } peers { permanent_uuid: "edb0578f53e94642998053acdf451eaa" member_type: VOTER last_known_addr { host: "127.4.25.61" port: 34743 } } peers { permanent_uuid: "83cb24a13df94c81ab682e2991154bb8" member_type: VOTER last_known_addr { host: "127.4.25.60" port: 41871 } }
15:07:18.766 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x6927bb61, L:/127.0.0.1:40458 - R:/127.4.25.62:46827] using SASL/GSSAPI
15:07:18.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.766144 4740 sys_catalog.cc:564] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa [sys.catalog]: configured and running, proceeding with master startup.
15:07:18.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.768276 4748 sys_catalog.cc:455] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9c71ce8e09a4f17be3147a4299ff704" member_type: VOTER last_known_addr { host: "127.4.25.62" port: 46827 } } peers { permanent_uuid: "edb0578f53e94642998053acdf451eaa" member_type: VOTER last_known_addr { host: "127.4.25.61" port: 34743 } } peers { permanent_uuid: "83cb24a13df94c81ab682e2991154bb8" member_type: VOTER last_known_addr { host: "127.4.25.60" port: 41871 } } }
15:07:18.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.768651 4748 sys_catalog.cc:458] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa [sys.catalog]: This master's current role is: FOLLOWER
15:07:18.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 12 15:07:18 dist-test-slave-kjk3 krb5kdc[4206](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736694435, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.4.25.60@KRBTEST.COM
15:07:18.775 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x846e9696, L:/127.0.0.1:46282 - R:/127.4.25.61:34743] using SASL/GSSAPI
15:07:18.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.777204 4759 catalog_manager.cc:1260] Loaded cluster ID: d8ef055024ef43fb9fd0393c750b5864
15:07:18.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.777263 4759 catalog_manager.cc:1553] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa: loading cluster ID for follower catalog manager: success
15:07:18.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 12 15:07:18 dist-test-slave-kjk3 krb5kdc[4206](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736694438, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.25.61@KRBTEST.COM for kudu/127.4.25.62@KRBTEST.COM
15:07:18.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.779456 4759 catalog_manager.cc:1575] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa: acquiring CA information for follower catalog manager: success
15:07:18.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.781036 4759 catalog_manager.cc:1603] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:07:18.782 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x94397144, L:/127.0.0.1:43898 - R:/127.4.25.60:41871] using SASL/GSSAPI
15:07:18.785 [WARN - kudu-nio-0] (ConnectToCluster.java:302) None of the provided masters 127.4.25.60:41871,127.4.25.62:46827,127.4.25.61:34743 is a leader; will retry
15:07:18.794 [WARN - kudu-nio-0] (ConnectToCluster.java:302) None of the provided masters 127.4.25.60:41871,127.4.25.62:46827,127.4.25.61:34743 is a leader; will retry
15:07:18.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.800666 4656 master_runner.cc:263] No leader master found from master edb0578f53e94642998053acdf451eaa
15:07:18.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:18.800767 4656 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:07:18.813 [WARN - kudu-nio-0] (ConnectToCluster.java:302) None of the provided masters 127.4.25.60:41871,127.4.25.62:46827,127.4.25.61:34743 is a leader; will retry
15:07:18.834 [WARN - kudu-nio-0] (ConnectToCluster.java:302) None of the provided masters 127.4.25.60:41871,127.4.25.62:46827,127.4.25.61:34743 is a leader; will retry
15:07:18.853 [WARN - kudu-nio-0] (ConnectToCluster.java:302) None of the provided masters 127.4.25.60:41871,127.4.25.62:46827,127.4.25.61:34743 is a leader; will retry
15:07:18.873 [WARN - kudu-nio-0] (ConnectToCluster.java:302) None of the provided masters 127.4.25.60:41871,127.4.25.62:46827,127.4.25.61:34743 is a leader; will retry
15:07:18.892 [WARN - kudu-nio-0] (ConnectToCluster.java:302) None of the provided masters 127.4.25.60:41871,127.4.25.62:46827,127.4.25.61:34743 is a leader; will retry
15:07:18.973 [WARN - kudu-nio-0] (ConnectToCluster.java:302) None of the provided masters 127.4.25.60:41871,127.4.25.62:46827,127.4.25.61:34743 is a leader; will retry
15:07:19.112 [WARN - kudu-nio-0] (ConnectToCluster.java:302) None of the provided masters 127.4.25.60:41871,127.4.25.62:46827,127.4.25.61:34743 is a leader; will retry
15:07:19.612 [WARN - kudu-nio-0] (ConnectToCluster.java:302) None of the provided masters 127.4.25.60:41871,127.4.25.62:46827,127.4.25.61:34743 is a leader; will retry
15:07:20.132 [WARN - kudu-nio-0] (ConnectToCluster.java:302) None of the provided masters 127.4.25.60:41871,127.4.25.62:46827,127.4.25.61:34743 is a leader; will retry
15:07:20.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:20.151496 4762 raft_consensus.cc:491] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:07:20.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:20.151695 4762 raft_consensus.cc:513] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9c71ce8e09a4f17be3147a4299ff704" member_type: VOTER last_known_addr { host: "127.4.25.62" port: 46827 } } peers { permanent_uuid: "edb0578f53e94642998053acdf451eaa" member_type: VOTER last_known_addr { host: "127.4.25.61" port: 34743 } } peers { permanent_uuid: "83cb24a13df94c81ab682e2991154bb8" member_type: VOTER last_known_addr { host: "127.4.25.60" port: 41871 } }
15:07:20.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:20.152973 4762 leader_election.cc:290] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers c9c71ce8e09a4f17be3147a4299ff704 (127.4.25.62:46827), 83cb24a13df94c81ab682e2991154bb8 (127.4.25.60:41871)
15:07:20.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:20.153466 4543 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "edb0578f53e94642998053acdf451eaa" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "83cb24a13df94c81ab682e2991154bb8" is_pre_election: true
15:07:20.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:20.153491 4628 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "edb0578f53e94642998053acdf451eaa" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "c9c71ce8e09a4f17be3147a4299ff704" is_pre_election: true
15:07:20.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:20.153764 4543 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 83cb24a13df94c81ab682e2991154bb8 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate edb0578f53e94642998053acdf451eaa in term 1.
15:07:20.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:20.153788 4628 raft_consensus.cc:2463] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate edb0578f53e94642998053acdf451eaa in term 1.
15:07:20.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:20.154176 4691 leader_election.cc:304] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 3 yes votes; 0 no votes. yes voters: 83cb24a13df94c81ab682e2991154bb8, c9c71ce8e09a4f17be3147a4299ff704, edb0578f53e94642998053acdf451eaa; no voters:
15:07:20.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:20.154487 4762 raft_consensus.cc:2798] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa [term 1 FOLLOWER]: Leader pre-election won for term 2
15:07:20.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:20.154579 4762 raft_consensus.cc:491] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
15:07:20.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:20.154613 4762 raft_consensus.cc:3054] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa [term 1 FOLLOWER]: Advancing to term 2
15:07:20.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:20.156342 4762 raft_consensus.cc:513] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9c71ce8e09a4f17be3147a4299ff704" member_type: VOTER last_known_addr { host: "127.4.25.62" port: 46827 } } peers { permanent_uuid: "edb0578f53e94642998053acdf451eaa" member_type: VOTER last_known_addr { host: "127.4.25.61" port: 34743 } } peers { permanent_uuid: "83cb24a13df94c81ab682e2991154bb8" member_type: VOTER last_known_addr { host: "127.4.25.60" port: 41871 } }
15:07:20.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:20.156672 4762 leader_election.cc:290] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa [CANDIDATE]: Term 2 election: Requested vote from peers c9c71ce8e09a4f17be3147a4299ff704 (127.4.25.62:46827), 83cb24a13df94c81ab682e2991154bb8 (127.4.25.60:41871)
15:07:20.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:20.157027 4628 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "edb0578f53e94642998053acdf451eaa" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "c9c71ce8e09a4f17be3147a4299ff704"
15:07:20.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:20.157094 4543 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "edb0578f53e94642998053acdf451eaa" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "83cb24a13df94c81ab682e2991154bb8"
15:07:20.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:20.157142 4628 raft_consensus.cc:3054] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704 [term 1 FOLLOWER]: Advancing to term 2
15:07:20.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:20.157208 4543 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 83cb24a13df94c81ab682e2991154bb8 [term 1 FOLLOWER]: Advancing to term 2
15:07:20.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:20.158700 4543 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 83cb24a13df94c81ab682e2991154bb8 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate edb0578f53e94642998053acdf451eaa in term 2.
15:07:20.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:20.158702 4628 raft_consensus.cc:2463] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate edb0578f53e94642998053acdf451eaa in term 2.
15:07:20.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:20.158979 4691 leader_election.cc:304] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 83cb24a13df94c81ab682e2991154bb8, edb0578f53e94642998053acdf451eaa; no voters:
15:07:20.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:20.159148 4762 raft_consensus.cc:2798] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa [term 2 FOLLOWER]: Leader election won for term 2
15:07:20.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:20.159315 4762 raft_consensus.cc:695] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa [term 2 LEADER]: Becoming Leader. State: Replica: edb0578f53e94642998053acdf451eaa, State: Running, Role: LEADER
15:07:20.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:20.159628 4762 consensus_queue.cc:237] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9c71ce8e09a4f17be3147a4299ff704" member_type: VOTER last_known_addr { host: "127.4.25.62" port: 46827 } } peers { permanent_uuid: "edb0578f53e94642998053acdf451eaa" member_type: VOTER last_known_addr { host: "127.4.25.61" port: 34743 } } peers { permanent_uuid: "83cb24a13df94c81ab682e2991154bb8" member_type: VOTER last_known_addr { host: "127.4.25.60" port: 41871 } }
15:07:20.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:20.161398 4763 sys_catalog.cc:455] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa [sys.catalog]: SysCatalogTable state changed. Reason: New leader edb0578f53e94642998053acdf451eaa. Latest consensus state: current_term: 2 leader_uuid: "edb0578f53e94642998053acdf451eaa" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9c71ce8e09a4f17be3147a4299ff704" member_type: VOTER last_known_addr { host: "127.4.25.62" port: 46827 } } peers { permanent_uuid: "edb0578f53e94642998053acdf451eaa" member_type: VOTER last_known_addr { host: "127.4.25.61" port: 34743 } } peers { permanent_uuid: "83cb24a13df94c81ab682e2991154bb8" member_type: VOTER last_known_addr { host: "127.4.25.60" port: 41871 } } }
15:07:20.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:20.161507 4763 sys_catalog.cc:458] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa [sys.catalog]: This master's current role is: LEADER
15:07:20.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:20.161757 4765 catalog_manager.cc:1476] Loading table and tablet metadata into memory...
15:07:20.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:20.162626 4765 catalog_manager.cc:1485] Initializing Kudu cluster ID...
15:07:20.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:20.162942 4765 catalog_manager.cc:1260] Loaded cluster ID: d8ef055024ef43fb9fd0393c750b5864
15:07:20.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:20.162986 4765 catalog_manager.cc:1496] Initializing Kudu internal certificate authority...
15:07:20.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:20.163475 4765 catalog_manager.cc:1505] Loading token signing keys...
15:07:20.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:20.163825 4765 catalog_manager.cc:5910] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa: Loaded TSK: 0
15:07:20.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:20.164491 4765 catalog_manager.cc:1515] Initializing in-progress tserver states...
15:07:20.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:20.666558 4543 raft_consensus.cc:1270] T 00000000000000000000000000000000 P 83cb24a13df94c81ab682e2991154bb8 [term 2 FOLLOWER]: Refusing update from remote peer edb0578f53e94642998053acdf451eaa: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
15:07:20.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:20.667229 4766 consensus_queue.cc:1035] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa [LEADER]: Connected to new peer: Peer: permanent_uuid: "83cb24a13df94c81ab682e2991154bb8" member_type: VOTER last_known_addr { host: "127.4.25.60" port: 41871 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
15:07:20.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:20.669291 4767 sys_catalog.cc:455] T 00000000000000000000000000000000 P 83cb24a13df94c81ab682e2991154bb8 [sys.catalog]: SysCatalogTable state changed. Reason: New leader edb0578f53e94642998053acdf451eaa. Latest consensus state: current_term: 2 leader_uuid: "edb0578f53e94642998053acdf451eaa" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9c71ce8e09a4f17be3147a4299ff704" member_type: VOTER last_known_addr { host: "127.4.25.62" port: 46827 } } peers { permanent_uuid: "edb0578f53e94642998053acdf451eaa" member_type: VOTER last_known_addr { host: "127.4.25.61" port: 34743 } } peers { permanent_uuid: "83cb24a13df94c81ab682e2991154bb8" member_type: VOTER last_known_addr { host: "127.4.25.60" port: 41871 } } }
15:07:20.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:20.669404 4767 sys_catalog.cc:458] T 00000000000000000000000000000000 P 83cb24a13df94c81ab682e2991154bb8 [sys.catalog]: This master's current role is: FOLLOWER
15:07:20.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:20.670820 4769 sys_catalog.cc:455] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "edb0578f53e94642998053acdf451eaa" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9c71ce8e09a4f17be3147a4299ff704" member_type: VOTER last_known_addr { host: "127.4.25.62" port: 46827 } } peers { permanent_uuid: "edb0578f53e94642998053acdf451eaa" member_type: VOTER last_known_addr { host: "127.4.25.61" port: 34743 } } peers { permanent_uuid: "83cb24a13df94c81ab682e2991154bb8" member_type: VOTER last_known_addr { host: "127.4.25.60" port: 41871 } } }
15:07:20.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:20.671208 4767 sys_catalog.cc:455] T 00000000000000000000000000000000 P 83cb24a13df94c81ab682e2991154bb8 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "edb0578f53e94642998053acdf451eaa" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9c71ce8e09a4f17be3147a4299ff704" member_type: VOTER last_known_addr { host: "127.4.25.62" port: 46827 } } peers { permanent_uuid: "edb0578f53e94642998053acdf451eaa" member_type: VOTER last_known_addr { host: "127.4.25.61" port: 34743 } } peers { permanent_uuid: "83cb24a13df94c81ab682e2991154bb8" member_type: VOTER last_known_addr { host: "127.4.25.60" port: 41871 } } }
15:07:20.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:20.671272 4769 sys_catalog.cc:458] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa [sys.catalog]: This master's current role is: LEADER
15:07:20.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:20.671309 4767 sys_catalog.cc:458] T 00000000000000000000000000000000 P 83cb24a13df94c81ab682e2991154bb8 [sys.catalog]: This master's current role is: FOLLOWER
15:07:20.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:20.671510 4628 raft_consensus.cc:1270] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704 [term 2 FOLLOWER]: Refusing update from remote peer edb0578f53e94642998053acdf451eaa: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
15:07:20.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:20.672019 4769 consensus_queue.cc:1035] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa [LEADER]: Connected to new peer: Peer: permanent_uuid: "c9c71ce8e09a4f17be3147a4299ff704" member_type: VOTER last_known_addr { host: "127.4.25.62" port: 46827 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
15:07:20.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:20.673833 4773 sys_catalog.cc:455] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704 [sys.catalog]: SysCatalogTable state changed. Reason: New leader edb0578f53e94642998053acdf451eaa. Latest consensus state: current_term: 2 leader_uuid: "edb0578f53e94642998053acdf451eaa" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9c71ce8e09a4f17be3147a4299ff704" member_type: VOTER last_known_addr { host: "127.4.25.62" port: 46827 } } peers { permanent_uuid: "edb0578f53e94642998053acdf451eaa" member_type: VOTER last_known_addr { host: "127.4.25.61" port: 34743 } } peers { permanent_uuid: "83cb24a13df94c81ab682e2991154bb8" member_type: VOTER last_known_addr { host: "127.4.25.60" port: 41871 } } }
15:07:20.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:20.673853 4776 sys_catalog.cc:455] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "edb0578f53e94642998053acdf451eaa" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9c71ce8e09a4f17be3147a4299ff704" member_type: VOTER last_known_addr { host: "127.4.25.62" port: 46827 } } peers { permanent_uuid: "edb0578f53e94642998053acdf451eaa" member_type: VOTER last_known_addr { host: "127.4.25.61" port: 34743 } } peers { permanent_uuid: "83cb24a13df94c81ab682e2991154bb8" member_type: VOTER last_known_addr { host: "127.4.25.60" port: 41871 } } }
15:07:20.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:20.673936 4773 sys_catalog.cc:458] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704 [sys.catalog]: This master's current role is: FOLLOWER
15:07:20.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:20.673951 4776 sys_catalog.cc:458] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704 [sys.catalog]: This master's current role is: FOLLOWER
15:07:21.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:21.165609 4769 sys_catalog.cc:455] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "edb0578f53e94642998053acdf451eaa" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9c71ce8e09a4f17be3147a4299ff704" member_type: VOTER last_known_addr { host: "127.4.25.62" port: 46827 } } peers { permanent_uuid: "edb0578f53e94642998053acdf451eaa" member_type: VOTER last_known_addr { host: "127.4.25.61" port: 34743 } } peers { permanent_uuid: "83cb24a13df94c81ab682e2991154bb8" member_type: VOTER last_known_addr { host: "127.4.25.60" port: 41871 } } }
15:07:21.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:21.165787 4769 sys_catalog.cc:458] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa [sys.catalog]: This master's current role is: LEADER
15:07:21.354 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
15:07:21.355 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=12, TimeoutTracker(timeout=30000, elapsed=2625), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.25.60:41871, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.25.62:46827, [3ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.25.61:34743, [38ms] Sub RPC ConnectToMaster: received response from server master-127.4.25.62:46827: OK, [49ms] Sub RPC ConnectToMaster: received response from server master-127.4.25.61:34743: OK, [55ms] Sub RPC ConnectToMaster: received response from server master-127.4.25.60:41871: OK, [56ms] delaying RPC due to: Service unavailable: Master config (127.4.25.60:41871,127.4.25.62:46827,127.4.25.61:34743) has no leader., [61ms] refreshing cache from master, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.25.60:41871, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.25.62:46827, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.25.61:34743, [63ms] Sub RPC ConnectToMaster: received response from server master-127.4.25.62:46827: OK, [64ms] Sub RPC ConnectToMaster: received response from server master-127.4.25.60:41871: OK, [65ms] Sub RPC ConnectToMaster: received response from server master-127.4.25.61:34743: OK, [65ms] delaying RPC due to: Service unavailable: Master config (127.4.25.60:41871,127.4.25.62:46827,127.4.25.61:34743) has no leader., [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.25.60:41871, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.25.62:46827, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.25.61:34743, [83ms] Sub RPC ConnectToMaster: received response from server master-127.4.25.62:46827: OK, [83ms] Sub RPC ConnectToMaster: received response from server master-127.4.25.60:41871: OK, [83ms] Sub RPC ConnectToMaster: received response from server master-127.4.25.61:34743: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.4.25.60:41871,127.4.25.62:46827,127.4.25.61:34743) has no leader., [102ms] refreshing cache from master, [102ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.25.60:41871, [102ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.25.62:46827, [102ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.25.61:34743, [103ms] Sub RPC ConnectToMaster: received response from server master-127.4.25.62:46827: OK, [104ms] Sub RPC ConnectToMaster: received response from server master-127.4.25.60:41871: OK, [104ms] Sub RPC ConnectToMaster: received response from server master-127.4.25.61:34743: OK, [105ms] delaying RPC due to: Service unavailable: Master config (127.4.25.60:41871,127.4.25.62:46827,127.4.25.61:34743) has no leader., [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.25.60:41871, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.25.62:46827, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.25.61:34743, [124ms] Sub RPC ConnectToMaster: received response from server master-127.4.25.60:41871: OK, [124ms] Sub RPC ConnectToMaster: received response from server master-127.4.25.62:46827: OK, [124ms] Sub RPC ConnectToMaster: received response from server master-127.4.25.61:34743: OK, [124ms] delaying RPC due to: Service unavailable: Master config (127.4.25.60:41871,127.4.25.62:46827,127.4.25.61:34743) has no leader., [141ms] refreshing cache from master, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.25.60:41871, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.25.62:46827, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.25.61:34743, [143ms] Sub RPC ConnectToMaster: received response from server master-127.4.25.62:46827: OK, [143ms] Sub RPC ConnectToMaster: received response from server master-127.4.25.60:41871: OK, [144ms] Sub RPC ConnectToMaster: received response from server master-127.4.25.61:34743: OK, [144ms] delaying RPC due to: Service unavailable: Master config (127.4.25.60:41871,127.4.25.62:46827,127.4.25.61:34743) has no leader., [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.25.60:41871, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.25.62:46827, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.25.61:34743, [163ms] Sub RPC ConnectToMaster: received response from server master-127.4.25.61:34743: OK, [163ms] Sub RPC ConnectToMaster: received response from server master-127.4.25.62:46827: OK, [163ms] Sub RPC ConnectToMaster: received response from server master-127.4.25.60:41871: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.4.25.60:41871,127.4.25.62:46827,127.4.25.61:34743) has no leader., [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.25.60:41871, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.25.62:46827, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.25.61:34743, [243ms] Sub RPC ConnectToMaster: received response from server master-127.4.25.62:46827: OK, [243ms] Sub RPC ConnectToMaster: received response from server master-127.4.25.60:41871: OK, [243ms] Sub RPC ConnectToMaster: received response from server master-127.4.25.61:34743: OK, [244ms] delaying RPC due to: Service unavailable: Master config (127.4.25.60:41871,127.4.25.62:46827,127.4.25.61:34743) has no leader., [381ms] refreshing cache from master, [381ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.25.60:41871, [382ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.25.62:46827, [382ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.25.61:34743, [383ms] Sub RPC ConnectToMaster: received response from server master-127.4.25.60:41871: OK, [383ms] Sub RPC ConnectToMaster: received response from server master-127.4.25.62:46827: OK, [383ms] Sub RPC ConnectToMaster: received response from server master-127.4.25.61:34743: OK, [384ms] delaying RPC due to: Service unavailable: Master config (127.4.25.60:41871,127.4.25.62:46827,127.4.25.61:34743) has no leader., [881ms] refreshing cache from master, [881ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.25.60:41871, [881ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.25.62:46827, [881ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.25.61:34743, [882ms] Sub RPC ConnectToMaster: received response from server master-127.4.25.60:41871: OK, [882ms] Sub RPC ConnectToMaster: received response from server master-127.4.25.62:46827: OK, [882ms] Sub RPC ConnectToMaster: received response from server master-127.4.25.61:34743: OK, [883ms] delaying RPC due to: Service unavailable: Master config (127.4.25.60:41871,127.4.25.62:46827,127.4.25.61:34743) has no leader., [1401ms] refreshing cache from master, [1401ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.25.60:41871, [1401ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.25.62:46827, [1401ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.25.61:34743, [1403ms] Sub RPC ConnectToMaster: received response from server master-127.4.25.60:41871: OK, [1403ms] Sub RPC ConnectToMaster: received response from server master-127.4.25.62:46827: OK, [1403ms] Sub RPC ConnectToMaster: received response from server master-127.4.25.61:34743: OK, [1403ms] delaying RPC due to: Service unavailable: Master config (127.4.25.60:41871,127.4.25.62:46827,127.4.25.61:34743) has no leader., [2621ms] refreshing cache from master, [2621ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.25.60:41871, [2622ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.25.62:46827, [2622ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.25.61:34743, [2623ms] Sub RPC ConnectToMaster: received response from server master-127.4.25.62:46827: OK, [2623ms] Sub RPC ConnectToMaster: received response from server master-127.4.25.60:41871: OK, [2625ms] Sub RPC ConnectToMaster: received response from server master-127.4.25.61:34743: OK, deferred=Deferred@1426541106(state=PENDING, result=null, callback=(continuation of Deferred@2036859393 after retry RPC after error@143402160) -> (continuation of Deferred@14582866 after retry RPC after error@1877416097) -> (continuation of Deferred@1453750102 after retry RPC after error@1063674006) -> (continuation of Deferred@1273929532 after retry RPC after error@471577840) -> (continuation of Deferred@2044882146 after retry RPC after error@1963770264) -> (continuation of Deferred@1252304762 after retry RPC after error@198784746) -> (continuation of Deferred@1639370255 after retry RPC after error@1923470240) -> (continuation of Deferred@500508153 after retry RPC after error@1601777359) -> (continuation of Deferred@149439897 after retry RPC after error@1410121719) -> (continuation of Deferred@1614979682 after retry RPC after error@1985734643) -> (continuation of Deferred@1012248620 after retry RPC after error@228016982), errback=(continuation of Deferred@2036859393 after retry RPC after error@143402160) -> (continuation of Deferred@14582866 after retry RPC after error@1877416097) -> (continuation of Deferred@1453750102 after retry RPC after error@1063674006) -> (continuation of Deferred@1273929532 after retry RPC after error@471577840) -> (continuation of Deferred@2044882146 after retry RPC after error@1963770264) -> (continuation of Deferred@1252304762 after retry RPC after error@198784746) -> (continuation of Deferred@1639370255 after retry RPC after error@1923470240) -> (continuation of Deferred@500508153 after retry RPC after error@1601777359) -> (continuation of Deferred@149439897 after retry RPC after error@1410121719) -> (continuation of Deferred@1614979682 after retry RPC after error@1985734643) -> (continuation of Deferred@1012248620 after retry RPC after error@228016982))) after lookup
15:07:24.408 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.4.25.60:41871
15:07:24.409 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
15:07:24.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.409281 4196 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskco6SfS/build/debug/bin/kudu with pid 4501
15:07:24.415 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:07:24.415 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.4.25.62:46827
15:07:24.415 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.4.25.60:41871(127.4.25.60:41871)] handling channelInactive
15:07:24.415 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
15:07:24.415 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.25.60:41871(127.4.25.60:41871)] cleaning up while in state READY due to: connection closed
15:07:24.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.415856 4196 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskco6SfS/build/debug/bin/kudu with pid 4571
15:07:24.421 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:07:24.421 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.4.25.62:46827(127.4.25.62:46827)] handling channelInactive
15:07:24.421 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.4.25.61:34743
15:07:24.421 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.25.62:46827(127.4.25.62:46827)] cleaning up while in state READY due to: connection closed
15:07:24.421 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
15:07:24.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.421612 4196 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskco6SfS/build/debug/bin/kudu with pid 4656
15:07:24.426 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:07:24.426 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.4.25.61:34743(127.4.25.61:34743)] handling channelInactive
15:07:24.426 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.4.25.60:41871
15:07:24.426 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.25.61:34743(127.4.25.61:34743)] cleaning up while in state READY due to: connection closed
15:07:24.426 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
15:07:24.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.427409 4196 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskco6SfS/build/debug/bin/kudu
15:07:24.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskco6SfS/build/debug/bin/kudu
15:07:24.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-2/wal
15:07:24.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-2/data
15:07:24.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:07:24.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:07:24.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:07:24.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:07:24.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:07:24.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:07:24.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-2/logs
15:07:24.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-2/data/info.pb
15:07:24.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:07:24.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:07:24.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:07:24.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:07:24.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:07:24.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:07:24.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:07:24.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:07:24.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:07:24.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:07:24.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:07:24.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:07:24.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.25.60:41871
15:07:24.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.25.60
15:07:24.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33051
15:07:24.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/krb5kdc/kudu_127.4.25.60.keytab
15:07:24.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.25.60
15:07:24.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:07:24.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:07:24.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:07:24.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.25.20:33809
15:07:24.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:07:24.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:07:24.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:07:24.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:07:24.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.25.62:46827,127.4.25.61:34743,127.4.25.60:41871
15:07:24.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:07:24.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34843/jwks.json
15:07:24.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:07:24.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:24.531985 4786 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:07:24.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:24.532253 4786 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:07:24.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:24.532325 4786 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:07:24.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:24.535341 4786 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:07:24.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:24.535396 4786 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:07:24.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:24.535421 4786 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:07:24.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:24.535439 4786 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:07:24.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:24.535454 4786 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:07:24.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:24.535470 4786 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:07:24.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:24.535483 4786 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34843/jwks.json
15:07:24.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.539762 4786 master_runner.cc:386] Master server non-default flags:
15:07:24.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:07:24.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.25.20:33809
15:07:24.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:07:24.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:07:24.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-2/data
15:07:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-2/wal
15:07:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:07:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:07:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.25.62:46827,127.4.25.61:34743,127.4.25.60:41871
15:07:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/krb5kdc/kudu_127.4.25.60.keytab
15:07:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.25.60
15:07:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:07:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:07:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:07:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.25.60:41871
15:07:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:07:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:07:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:07:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34843/jwks.json
15:07:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:07:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:07:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:07:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:07:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:07:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-2/data/info.pb
15:07:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.25.60
15:07:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33051
15:07:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:07:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.4786
15:07:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:07:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:07:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:07:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:07:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-2/logs
15:07:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:07:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:07:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:07:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:07:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:07:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:07:24.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
15:07:24.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Jan 2025 14:43:14 UTC on 24a791456cd2
15:07:24.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4141
15:07:24.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.540797 4786 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:07:24.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.541591 4786 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:07:24.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:24.548568 4791 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:07:24.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:24.549254 4794 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:07:24.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:24.549783 4792 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:07:24.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.550655 4786 server_base.cc:1034] running on GCE node
15:07:24.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.551168 4786 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:07:24.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.552086 4786 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:07:24.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.553259 4786 hybrid_clock.cc:648] HybridClock initialized: now 1736694444553234 us; error 46 us; skew 500 ppm
15:07:24.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 12 15:07:24 dist-test-slave-kjk3 krb5kdc[4206](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736694444, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.25.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:07:24.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.559811 4786 init.cc:377] Logged in from keytab as kudu/127.4.25.60@KRBTEST.COM (short username kudu)
15:07:24.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.560745 4786 webserver.cc:458] Webserver started at http://127.4.25.60:33051/ using document root <none> and password file <none>
15:07:24.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.561241 4786 fs_manager.cc:362] Metadata directory not provided
15:07:24.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.561288 4786 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:07:24.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.564675 4786 fs_manager.cc:714] Time spent opening directory manager: real 0.002s user 0.003s sys 0.000s
15:07:24.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.566462 4801 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:07:24.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.567468 4786 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
15:07:24.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.567582 4786 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-2/data,/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-2/wal
15:07:24.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "83cb24a13df94c81ab682e2991154bb8"
15:07:24.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:07:14 on dist-test-slave-kjk3"
15:07:24.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.567975 4786 fs_report.cc:389] FS layout report
15:07:24.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:07:24.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-2/wal
15:07:24.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-2/wal
15:07:24.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-2/data/data
15:07:24.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:07:24.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:07:24.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:07:24.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:07:24.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:07:24.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:07:24.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:07:24.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:07:24.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:07:24.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:07:24.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:07:24.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.587417 4786 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:07:24.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.590085 4786 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:07:24.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.590232 4786 kserver.cc:163] Server-wide thread pool size limit: 3276
15:07:24.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.596406 4786 rpc_server.cc:307] RPC server started. Bound to: 127.4.25.60:41871
15:07:24.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.596410 4853 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.4.25.60:41871 every 8 connection(s)
15:07:24.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.597448 4786 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-2/data/info.pb
15:07:24.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.601397 4854 sys_catalog.cc:263] Verifying existing consensus state
15:07:24.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.602551 4196 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskco6SfS/build/debug/bin/kudu as pid 4786
15:07:24.602 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:07:24.602 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.4.25.62:46827
15:07:24.603 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
15:07:24.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.603031 4854 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 83cb24a13df94c81ab682e2991154bb8: Bootstrap starting.
15:07:24.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.603406 4196 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskco6SfS/build/debug/bin/kudu
15:07:24.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskco6SfS/build/debug/bin/kudu
15:07:24.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-0/wal
15:07:24.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-0/data
15:07:24.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:07:24.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:07:24.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:07:24.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:07:24.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:07:24.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:07:24.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-0/logs
15:07:24.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-0/data/info.pb
15:07:24.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:07:24.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:07:24.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:07:24.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:07:24.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:07:24.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:07:24.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:07:24.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:07:24.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:07:24.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:07:24.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:07:24.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:07:24.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.25.62:46827
15:07:24.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.25.62
15:07:24.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35181
15:07:24.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/krb5kdc/kudu_127.4.25.62.keytab
15:07:24.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.25.62
15:07:24.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:07:24.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:07:24.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:07:24.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.25.20:33809
15:07:24.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:07:24.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:07:24.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:07:24.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:07:24.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.25.62:46827,127.4.25.61:34743,127.4.25.60:41871
15:07:24.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:07:24.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34843/jwks.json
15:07:24.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:07:24.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.616158 4854 log.cc:826] T 00000000000000000000000000000000 P 83cb24a13df94c81ab682e2991154bb8: Log is configured to *not* fsync() on all Append() calls
15:07:24.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.623265 4854 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 83cb24a13df94c81ab682e2991154bb8: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
15:07:24.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.623634 4854 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 83cb24a13df94c81ab682e2991154bb8: Bootstrap complete.
15:07:24.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.627931 4854 raft_consensus.cc:357] T 00000000000000000000000000000000 P 83cb24a13df94c81ab682e2991154bb8 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9c71ce8e09a4f17be3147a4299ff704" member_type: VOTER last_known_addr { host: "127.4.25.62" port: 46827 } } peers { permanent_uuid: "edb0578f53e94642998053acdf451eaa" member_type: VOTER last_known_addr { host: "127.4.25.61" port: 34743 } } peers { permanent_uuid: "83cb24a13df94c81ab682e2991154bb8" member_type: VOTER last_known_addr { host: "127.4.25.60" port: 41871 } }
15:07:24.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.628365 4854 raft_consensus.cc:738] T 00000000000000000000000000000000 P 83cb24a13df94c81ab682e2991154bb8 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 83cb24a13df94c81ab682e2991154bb8, State: Initialized, Role: FOLLOWER
15:07:24.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.628885 4854 consensus_queue.cc:260] T 00000000000000000000000000000000 P 83cb24a13df94c81ab682e2991154bb8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9c71ce8e09a4f17be3147a4299ff704" member_type: VOTER last_known_addr { host: "127.4.25.62" port: 46827 } } peers { permanent_uuid: "edb0578f53e94642998053acdf451eaa" member_type: VOTER last_known_addr { host: "127.4.25.61" port: 34743 } } peers { permanent_uuid: "83cb24a13df94c81ab682e2991154bb8" member_type: VOTER last_known_addr { host: "127.4.25.60" port: 41871 } }
15:07:24.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.629329 4858 sys_catalog.cc:455] T 00000000000000000000000000000000 P 83cb24a13df94c81ab682e2991154bb8 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9c71ce8e09a4f17be3147a4299ff704" member_type: VOTER last_known_addr { host: "127.4.25.62" port: 46827 } } peers { permanent_uuid: "edb0578f53e94642998053acdf451eaa" member_type: VOTER last_known_addr { host: "127.4.25.61" port: 34743 } } peers { permanent_uuid: "83cb24a13df94c81ab682e2991154bb8" member_type: VOTER last_known_addr { host: "127.4.25.60" port: 41871 } } }
15:07:24.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.629451 4858 sys_catalog.cc:458] T 00000000000000000000000000000000 P 83cb24a13df94c81ab682e2991154bb8 [sys.catalog]: This master's current role is: FOLLOWER
15:07:24.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.630055 4854 sys_catalog.cc:564] T 00000000000000000000000000000000 P 83cb24a13df94c81ab682e2991154bb8 [sys.catalog]: configured and running, proceeding with master startup.
15:07:24.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.634893 4869 catalog_manager.cc:1260] Loaded cluster ID: d8ef055024ef43fb9fd0393c750b5864
15:07:24.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.635092 4869 catalog_manager.cc:1553] T 00000000000000000000000000000000 P 83cb24a13df94c81ab682e2991154bb8: loading cluster ID for follower catalog manager: success
15:07:24.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.634976 4786 master_runner.cc:186] Error getting master registration for 127.4.25.62:46827: Network error: Client connection negotiation failed: client connection to 127.4.25.62:46827: connect: Connection refused (error 111),
15:07:24.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.637041 4786 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:07:24.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.638024 4869 catalog_manager.cc:1575] T 00000000000000000000000000000000 P 83cb24a13df94c81ab682e2991154bb8: acquiring CA information for follower catalog manager: success
15:07:24.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.639443 4869 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 83cb24a13df94c81ab682e2991154bb8: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:07:24.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:24.715896 4856 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:07:24.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:24.716147 4856 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:07:24.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:24.716220 4856 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:07:24.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:24.719437 4856 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:07:24.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:24.719493 4856 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:07:24.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:24.719528 4856 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:07:24.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:24.719550 4856 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:07:24.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:24.719571 4856 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:07:24.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:24.719594 4856 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:07:24.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:24.719610 4856 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34843/jwks.json
15:07:24.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.723932 4856 master_runner.cc:386] Master server non-default flags:
15:07:24.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:07:24.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.25.20:33809
15:07:24.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:07:24.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:07:24.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-0/data
15:07:24.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-0/wal
15:07:24.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:07:24.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:07:24.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.25.62:46827,127.4.25.61:34743,127.4.25.60:41871
15:07:24.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/krb5kdc/kudu_127.4.25.62.keytab
15:07:24.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.25.62
15:07:24.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:07:24.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:07:24.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:07:24.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.25.62:46827
15:07:24.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:07:24.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:07:24.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:07:24.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34843/jwks.json
15:07:24.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:07:24.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:07:24.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:07:24.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:07:24.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:07:24.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-0/data/info.pb
15:07:24.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.25.62
15:07:24.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35181
15:07:24.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:07:24.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.4856
15:07:24.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:07:24.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:07:24.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:07:24.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:07:24.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-0/logs
15:07:24.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:07:24.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:07:24.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:07:24.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:07:24.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:07:24.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:07:24.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
15:07:24.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Jan 2025 14:43:14 UTC on 24a791456cd2
15:07:24.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4141
15:07:24.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.725302 4856 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:07:24.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.726233 4856 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:07:24.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:24.733742 4876 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:07:24.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:24.734568 4879 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:07:24.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:24.735006 4877 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:07:24.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.736186 4856 server_base.cc:1034] running on GCE node
15:07:24.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.736816 4856 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:07:24.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.737937 4856 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:07:24.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.739135 4856 hybrid_clock.cc:648] HybridClock initialized: now 1736694444739123 us; error 40 us; skew 500 ppm
15:07:24.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 12 15:07:24 dist-test-slave-kjk3 krb5kdc[4206](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736694444, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.25.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:07:24.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.742517 4856 init.cc:377] Logged in from keytab as kudu/127.4.25.62@KRBTEST.COM (short username kudu)
15:07:24.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.743662 4856 webserver.cc:458] Webserver started at http://127.4.25.62:35181/ using document root <none> and password file <none>
15:07:24.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.744469 4856 fs_manager.cc:362] Metadata directory not provided
15:07:24.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.744544 4856 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:07:24.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.748769 4856 fs_manager.cc:714] Time spent opening directory manager: real 0.003s user 0.002s sys 0.000s
15:07:24.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.750752 4886 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:07:24.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.751847 4856 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
15:07:24.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.751955 4856 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-0/data,/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-0/wal
15:07:24.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c9c71ce8e09a4f17be3147a4299ff704"
15:07:24.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:07:14 on dist-test-slave-kjk3"
15:07:24.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.752454 4856 fs_report.cc:389] FS layout report
15:07:24.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:07:24.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-0/wal
15:07:24.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-0/wal
15:07:24.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-0/data/data
15:07:24.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:07:24.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:07:24.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:07:24.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:07:24.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:07:24.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:07:24.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:07:24.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:07:24.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:07:24.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:07:24.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:07:24.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.777884 4856 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:07:24.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.780963 4856 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:07:24.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.781134 4856 kserver.cc:163] Server-wide thread pool size limit: 3276
15:07:24.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.788134 4856 rpc_server.cc:307] RPC server started. Bound to: 127.4.25.62:46827
15:07:24.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.788172 4938 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.4.25.62:46827 every 8 connection(s)
15:07:24.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.789340 4856 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-0/data/info.pb
15:07:24.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.793258 4939 sys_catalog.cc:263] Verifying existing consensus state
15:07:24.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.794808 4939 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704: Bootstrap starting.
15:07:24.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.799151 4196 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskco6SfS/build/debug/bin/kudu as pid 4856
15:07:24.799 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:07:24.799 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.4.25.61:34743
15:07:24.799 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
15:07:24.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.800226 4196 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskco6SfS/build/debug/bin/kudu
15:07:24.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskco6SfS/build/debug/bin/kudu
15:07:24.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-1/wal
15:07:24.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-1/data
15:07:24.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:07:24.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:07:24.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:07:24.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:07:24.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:07:24.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:07:24.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-1/logs
15:07:24.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-1/data/info.pb
15:07:24.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:07:24.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:07:24.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:07:24.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:07:24.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:07:24.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:07:24.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:07:24.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:07:24.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:07:24.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:07:24.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:07:24.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:07:24.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.25.61:34743
15:07:24.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.25.61
15:07:24.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43665
15:07:24.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/krb5kdc/kudu_127.4.25.61.keytab
15:07:24.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.25.61
15:07:24.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:07:24.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:07:24.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:07:24.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.25.20:33809
15:07:24.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:07:24.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:07:24.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:07:24.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:07:24.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.25.62:46827,127.4.25.61:34743,127.4.25.60:41871
15:07:24.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:07:24.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34843/jwks.json
15:07:24.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:07:24.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.808131 4939 log.cc:826] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704: Log is configured to *not* fsync() on all Append() calls
15:07:24.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.814179 4939 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
15:07:24.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.814507 4939 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704: Bootstrap complete.
15:07:24.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.817965 4939 raft_consensus.cc:357] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9c71ce8e09a4f17be3147a4299ff704" member_type: VOTER last_known_addr { host: "127.4.25.62" port: 46827 } } peers { permanent_uuid: "edb0578f53e94642998053acdf451eaa" member_type: VOTER last_known_addr { host: "127.4.25.61" port: 34743 } } peers { permanent_uuid: "83cb24a13df94c81ab682e2991154bb8" member_type: VOTER last_known_addr { host: "127.4.25.60" port: 41871 } }
15:07:24.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.818300 4939 raft_consensus.cc:738] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: c9c71ce8e09a4f17be3147a4299ff704, State: Initialized, Role: FOLLOWER
15:07:24.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.818770 4939 consensus_queue.cc:260] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9c71ce8e09a4f17be3147a4299ff704" member_type: VOTER last_known_addr { host: "127.4.25.62" port: 46827 } } peers { permanent_uuid: "edb0578f53e94642998053acdf451eaa" member_type: VOTER last_known_addr { host: "127.4.25.61" port: 34743 } } peers { permanent_uuid: "83cb24a13df94c81ab682e2991154bb8" member_type: VOTER last_known_addr { host: "127.4.25.60" port: 41871 } }
15:07:24.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.819190 4943 sys_catalog.cc:455] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9c71ce8e09a4f17be3147a4299ff704" member_type: VOTER last_known_addr { host: "127.4.25.62" port: 46827 } } peers { permanent_uuid: "edb0578f53e94642998053acdf451eaa" member_type: VOTER last_known_addr { host: "127.4.25.61" port: 34743 } } peers { permanent_uuid: "83cb24a13df94c81ab682e2991154bb8" member_type: VOTER last_known_addr { host: "127.4.25.60" port: 41871 } } }
15:07:24.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.819319 4943 sys_catalog.cc:458] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704 [sys.catalog]: This master's current role is: FOLLOWER
15:07:24.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.819823 4939 sys_catalog.cc:564] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704 [sys.catalog]: configured and running, proceeding with master startup.
15:07:24.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.824731 4954 catalog_manager.cc:1260] Loaded cluster ID: d8ef055024ef43fb9fd0393c750b5864
15:07:24.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.824973 4954 catalog_manager.cc:1553] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704: loading cluster ID for follower catalog manager: success
15:07:24.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.827584 4954 catalog_manager.cc:1575] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704: acquiring CA information for follower catalog manager: success
15:07:24.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.829237 4954 catalog_manager.cc:1603] T 00000000000000000000000000000000 P c9c71ce8e09a4f17be3147a4299ff704: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:07:24.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 12 15:07:24 dist-test-slave-kjk3 krb5kdc[4206](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736694444, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.25.62@KRBTEST.COM for kudu/127.4.25.62@KRBTEST.COM
15:07:24.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.840101 4856 master_runner.cc:186] Error getting master registration for 127.4.25.61:34743: Network error: Client connection negotiation failed: client connection to 127.4.25.61:34743: connect: Connection refused (error 111),
15:07:24.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.841837 4856 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:07:24.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:24.915606 4941 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:07:24.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:24.915827 4941 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:07:24.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:24.915882 4941 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:07:24.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:24.919129 4941 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:07:24.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:24.919188 4941 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:07:24.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:24.919214 4941 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:07:24.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:24.919231 4941 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:07:24.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:24.919246 4941 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:07:24.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:24.919270 4941 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:07:24.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:24.919283 4941 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34843/jwks.json
15:07:24.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.923708 4941 master_runner.cc:386] Master server non-default flags:
15:07:24.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:07:24.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.25.20:33809
15:07:24.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:07:24.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:07:24.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-1/data
15:07:24.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-1/wal
15:07:24.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:07:24.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:07:24.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.25.62:46827,127.4.25.61:34743,127.4.25.60:41871
15:07:24.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/krb5kdc/kudu_127.4.25.61.keytab
15:07:24.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.25.61
15:07:24.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:07:24.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:07:24.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:07:24.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.25.61:34743
15:07:24.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:07:24.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:07:24.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:07:24.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34843/jwks.json
15:07:24.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:07:24.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:07:24.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:07:24.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:07:24.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:07:24.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-1/data/info.pb
15:07:24.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.25.61
15:07:24.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43665
15:07:24.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:07:24.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.4941
15:07:24.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:07:24.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:07:24.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:07:24.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:07:24.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-1/logs
15:07:24.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:07:24.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:07:24.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:07:24.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:07:24.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:07:24.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:07:24.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type DEBUG
15:07:24.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Jan 2025 14:43:14 UTC on 24a791456cd2
15:07:24.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4141
15:07:24.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.924755 4941 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:07:24.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.925577 4941 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:07:24.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:24.933125 4962 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:07:24.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:24.933769 4963 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:07:24.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:24.934587 4965 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:07:24.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.935235 4941 server_base.cc:1034] running on GCE node
15:07:24.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.935724 4941 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:07:24.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.936677 4941 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:07:24.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.937840 4941 hybrid_clock.cc:648] HybridClock initialized: now 1736694444937825 us; error 26 us; skew 500 ppm
15:07:24.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 12 15:07:24 dist-test-slave-kjk3 krb5kdc[4206](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736694444, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.25.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:07:24.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.940752 4941 init.cc:377] Logged in from keytab as kudu/127.4.25.61@KRBTEST.COM (short username kudu)
15:07:24.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.941744 4941 webserver.cc:458] Webserver started at http://127.4.25.61:43665/ using document root <none> and password file <none>
15:07:24.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.942416 4941 fs_manager.cc:362] Metadata directory not provided
15:07:24.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.942479 4941 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:07:24.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.946167 4941 fs_manager.cc:714] Time spent opening directory manager: real 0.003s user 0.003s sys 0.000s
15:07:24.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.948197 4972 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:07:24.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.949321 4941 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
15:07:24.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.949425 4941 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-1/data,/tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-1/wal
15:07:24.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "edb0578f53e94642998053acdf451eaa"
15:07:24.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:07:14 on dist-test-slave-kjk3"
15:07:24.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.949837 4941 fs_report.cc:389] FS layout report
15:07:24.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:07:24.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-1/wal
15:07:24.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-1/wal
15:07:24.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-1/data/data
15:07:24.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:07:24.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:07:24.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:07:24.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:07:24.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:07:24.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:07:24.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:07:24.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:07:24.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:07:24.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:07:24.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:07:24.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.975613 4941 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:07:24.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.978772 4941 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:07:24.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.978935 4941 kserver.cc:163] Server-wide thread pool size limit: 3276
15:07:24.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.985868 4941 rpc_server.cc:307] RPC server started. Bound to: 127.4.25.61:34743
15:07:24.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.985900 5024 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.4.25.61:34743 every 8 connection(s)
15:07:24.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.986944 4941 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskco6SfS/test-tmp/mini-kudu-cluster3806855993863587601/master-1/data/info.pb
15:07:24.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.990953 5025 sys_catalog.cc:263] Verifying existing consensus state
15:07:24.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.992650 5025 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa: Bootstrap starting.
15:07:24.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:24.996352 4196 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskco6SfS/build/debug/bin/kudu as pid 4941
15:07:24.996 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:07:24.997 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.4.25.61:34743(127.4.25.61:34743)] connecting to peer
15:07:24.998 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.4.25.61:34743(127.4.25.61:34743)] Successfully connected to peer
15:07:25.001 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.4.25.61
15:07:25.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:25.006314 5025 log.cc:826] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa: Log is configured to *not* fsync() on all Append() calls
15:07:25.009 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.25.61:34743(127.4.25.61:34743)] cleaning up while in state NEGOTIATING due to: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
15:07:25.010 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.4.25.61:34743(127.4.25.61:34743)] disconnecting while in state TERMINATED
15:07:25.010 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.4.25.61:34743(127.4.25.61:34743)] handling channelInactive
15:07:25.010 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.4.25.62:46827(127.4.25.62:46827)] disconnecting while in state TERMINATED
15:07:25.011 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.4.25.60:41871(127.4.25.60:41871)] disconnecting while in state TERMINATED
15:07:25.011 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
15:07:25.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:07:25.011693 5027 negotiation.cc:337] Failed RPC negotiation. Trace:
15:07:25.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:07:24.998720 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:46328 (local address 127.4.25.61:34743)
15:07:25.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:07:24.999200 (+ 480us) server_negotiation.cc:206] Beginning negotiation
15:07:25.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:07:24.999212 (+ 12us) server_negotiation.cc:399] Waiting for connection header
15:07:25.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:07:24.999252 (+ 40us) server_negotiation.cc:407] Connection header received
15:07:25.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:07:24.999851 (+ 599us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
15:07:25.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:07:24.999866 (+ 15us) server_negotiation.cc:461] Received NEGOTIATE request from client
15:07:25.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:07:25.000232 (+ 366us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
15:07:25.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:07:25.000578 (+ 346us) server_negotiation.cc:226] Negotiated authn=SASL
15:07:25.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:07:25.003704 (+ 3126us) server_negotiation.cc:365] Received TLS_HANDSHAKE NegotiatePB request
15:07:25.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:07:25.005143 (+ 1439us) server_negotiation.cc:377] Sending TLS_HANDSHAKE NegotiatePB response
15:07:25.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:07:25.009429 (+ 4286us) server_negotiation.cc:365] Received TLS_HANDSHAKE NegotiatePB request
15:07:25.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:07:25.009741 (+ 312us) server_negotiation.cc:662] Negotiated TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 TLSv1.3 Kx=any Au=any Enc=AESGCM(128) Mac=AEAD
15:07:25.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:07:25.010831 (+ 1090us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:46328: BlockingRecv error: failed to read from TLS socket (remote: 127.0.0.1:46328): Cannot send after transport endpoint shutdown (error 108)
15:07:25.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":316,"thread_start_us":149,"threads_started":1}
15:07:25.013 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
15:07:25.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:25.013160 5025 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
15:07:25.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:25.013460 5025 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa: Bootstrap complete.
15:07:25.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:25.017513 5025 raft_consensus.cc:357] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9c71ce8e09a4f17be3147a4299ff704" member_type: VOTER last_known_addr { host: "127.4.25.62" port: 46827 } } peers { permanent_uuid: "edb0578f53e94642998053acdf451eaa" member_type: VOTER last_known_addr { host: "127.4.25.61" port: 34743 } } peers { permanent_uuid: "83cb24a13df94c81ab682e2991154bb8" member_type: VOTER last_known_addr { host: "127.4.25.60" port: 41871 } }
15:07:25.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:25.017979 5025 raft_consensus.cc:738] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: edb0578f53e94642998053acdf451eaa, State: Initialized, Role: FOLLOWER
15:07:25.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:25.018509 5025 consensus_queue.cc:260] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9c71ce8e09a4f17be3147a4299ff704" member_type: VOTER last_known_addr { host: "127.4.25.62" port: 46827 } } peers { permanent_uuid: "edb0578f53e94642998053acdf451eaa" member_type: VOTER last_known_addr { host: "127.4.25.61" port: 34743 } } peers { permanent_uuid: "83cb24a13df94c81ab682e2991154bb8" member_type: VOTER last_known_addr { host: "127.4.25.60" port: 41871 } }
15:07:25.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:25.018942 5032 sys_catalog.cc:455] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9c71ce8e09a4f17be3147a4299ff704" member_type: VOTER last_known_addr { host: "127.4.25.62" port: 46827 } } peers { permanent_uuid: "edb0578f53e94642998053acdf451eaa" member_type: VOTER last_known_addr { host: "127.4.25.61" port: 34743 } } peers { permanent_uuid: "83cb24a13df94c81ab682e2991154bb8" member_type: VOTER last_known_addr { host: "127.4.25.60" port: 41871 } } }
15:07:25.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:25.019043 5032 sys_catalog.cc:458] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa [sys.catalog]: This master's current role is: FOLLOWER
15:07:25.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:25.019770 5025 sys_catalog.cc:564] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa [sys.catalog]: configured and running, proceeding with master startup.
15:07:25.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:25.020838 4196 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskco6SfS/build/debug/bin/kudu with pid 4856
15:07:25.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:25.024888 5044 catalog_manager.cc:1260] Loaded cluster ID: d8ef055024ef43fb9fd0393c750b5864
15:07:25.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:25.024947 5044 catalog_manager.cc:1553] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa: loading cluster ID for follower catalog manager: success
15:07:25.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:25.027267 5044 catalog_manager.cc:1575] T 00000000000000000000000000000000 P edb0578f53e94642998053acdf451eaa: acquiring CA information for follower catalog manager: success
15:07:25.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:25.028005 4196 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskco6SfS/build/debug/bin/kudu with pid 4941
15:07:25.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:25.032349 4196 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskco6SfS/build/debug/bin/kudu with pid 4786
15:07:25.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-12T15:07:25Z chronyd exiting
15:07:25.082 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
15:07:25.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:25.083513 3505 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskco6SfS/build/debug/bin/kudu with pid 3790
15:07:25.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:25.088946 3505 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskco6SfS/build/debug/bin/kudu with pid 3925
15:07:25.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:25.094111 3505 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskco6SfS/build/debug/bin/kudu with pid 4060
15:07:25.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:25.100502 3505 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskco6SfS/build/debug/bin/kudu with pid 3516
15:07:25.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:25.106993 3505 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskco6SfS/build/debug/bin/kudu with pid 3587
15:07:25.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:07:25.112087 3505 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskco6SfS/build/debug/bin/kudu with pid 3658
15:07:25.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-12T15:07:25Z chronyd exiting
15:07:25.160 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectExpires: failed attempt 1
org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity$3.get(TestSecurity.java:386) ~[test/:?]
at org.apache.kudu.test.junit.AssertHelpers.assertEventuallyTrue(AssertHelpers.java:40) ~[kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.TestSecurity.assertEventualAuthenticationFailure(TestSecurity.java:378) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectExpires(TestSecurity.java:412) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.evaluateChallenge(Negotiator.java:974) ~[main/:?]
at org.apache.kudu.client.Negotiator.sendSaslInitiate(Negotiator.java:845) ~[main/:?]
at org.apache.kudu.client.Negotiator.startAuthentication(Negotiator.java:782) ~[main/:?]
at org.apache.kudu.client.Negotiator.handleTlsMessage(Negotiator.java:731) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:379) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Caused by: javax.security.sasl.SaslException: GSS initiate failed
at com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:211) ~[?:1.8.0_275]
at org.apache.kudu.client.Negotiator$1.run(Negotiator.java:959) ~[main/:?]
at org.apache.kudu.client.Negotiator$1.run(Negotiator.java:956) ~[main/:?]
at java.security.AccessController.doPrivileged(Native Method) ~[?:1.8.0_275]
at javax.security.auth.Subject.doAs(Subject.java:422) ~[?:1.8.0_275]
at org.apache.kudu.client.Negotiator.evaluateChallenge(Negotiator.java:955) ~[main/:?]
at org.apache.kudu.client.Negotiator.sendSaslInitiate(Negotiator.java:845) ~[main/:?]
at org.apache.kudu.client.Negotiator.startAuthentication(Negotiator.java:782) ~[main/:?]
at org.apache.kudu.client.Negotiator.handleTlsMessage(Negotiator.java:731) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:379) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
Caused by: org.ietf.jgss.GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)
at sun.security.jgss.krb5.Krb5InitCredential.getInstance(Krb5InitCredential.java:162) ~[?:1.8.0_275]
at sun.security.jgss.krb5.Krb5MechFactory.getCredentialElement(Krb5MechFactory.java:122) ~[?:1.8.0_275]
at sun.security.jgss.krb5.Krb5MechFactory.getMechanismContext(Krb5MechFactory.java:189) ~[?:1.8.0_275]
at sun.security.jgss.GSSManagerImpl.getMechanismContext(GSSManagerImpl.java:224) ~[?:1.8.0_275]
at sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:212) ~[?:1.8.0_275]
at sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:179) ~[?:1.8.0_275]
at com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:192) ~[?:1.8.0_275]
at org.apache.kudu.client.Negotiator$1.run(Negotiator.java:959) ~[main/:?]
at org.apache.kudu.client.Negotiator$1.run(Negotiator.java:956) ~[main/:?]
at java.security.AccessController.doPrivileged(Native Method) ~[?:1.8.0_275]
at javax.security.auth.Subject.doAs(Subject.java:422) ~[?:1.8.0_275]
at org.apache.kudu.client.Negotiator.evaluateChallenge(Negotiator.java:955) ~[main/:?]
at org.apache.kudu.client.Negotiator.sendSaslInitiate(Negotiator.java:845) ~[main/:?]
at org.apache.kudu.client.Negotiator.startAuthentication(Negotiator.java:782) ~[main/:?]
at org.apache.kudu.client.Negotiator.handleTlsMessage(Negotiator.java:731) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:379) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity$3.get(TestSecurity.java:386)
at org.apache.kudu.test.junit.AssertHelpers.assertEventuallyTrue(AssertHelpers.java:40)
at org.apache.kudu.client.TestSecurity.assertEventualAuthenticationFailure(TestSecurity.java:378)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectExpires(TestSecurity.java:412)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.evaluateChallenge(Negotiator.java:974)
at org.apache.kudu.client.Negotiator.sendSaslInitiate(Negotiator.java:845)
at org.apache.kudu.client.Negotiator.startAuthentication(Negotiator.java:782)
at org.apache.kudu.client.Negotiator.handleTlsMessage(Negotiator.java:731)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:379)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)]
at com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:211)
at org.apache.kudu.client.Negotiator$1.run(Negotiator.java:959)
at org.apache.kudu.client.Negotiator$1.run(Negotiator.java:956)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at org.apache.kudu.client.Negotiator.evaluateChallenge(Negotiator.java:955)
at org.apache.kudu.client.Negotiator.sendSaslInitiate(Negotiator.java:845)
at org.apache.kudu.client.Negotiator.startAuthentication(Negotiator.java:782)
at org.apache.kudu.client.Negotiator.handleTlsMessage(Negotiator.java:731)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:379)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)
at sun.security.jgss.krb5.Krb5InitCredential.getInstance(Krb5InitCredential.java:162)
at sun.security.jgss.krb5.Krb5MechFactory.getCredentialElement(Krb5MechFactory.java:122)
at sun.security.jgss.krb5.Krb5MechFactory.getMechanismContext(Krb5MechFactory.java:189)
at sun.security.jgss.GSSManagerImpl.getMechanismContext(GSSManagerImpl.java:224)
at sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:212)
at sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:179)
at com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:192)
... 42 more