04:02:44.274 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
04:02:44.285 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task4Qe23I/test-tmp
04:02:44.288 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
04:02:44.291 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task4Qe23I/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
04:02:44.502 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
04:02:44.656 [DEBUG - main] (MiniKuduCluster.java:195) Response:
04:02:44.657 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
04:02:44.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:44.660830 7113 mini_oidc.cc:152] Starting JWKS server
04:02:44.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:44.664953 7113 webserver.cc:460] Webserver started at http://127.0.0.1:44651/ using document root <none> and password file <none>
04:02:44.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:44.665438 7113 mini_oidc.cc:189] Starting OIDC Discovery server
04:02:44.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:44.667721 7113 webserver.cc:460] Webserver started at http://127.0.0.1:34841/ using document root <none> and password file <none>
04:02:44.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-28T04:02:44Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
04:02:44.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-28T04:02:44Z Disabled control of system clock
04:02:44.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:44.715684 7113 external_mini_cluster.cc:1326] Running /tmp/dist-test-task4Qe23I/build/tsan/bin/kudu
04:02:44.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4Qe23I/build/tsan/bin/kudu
04:02:44.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/master-0/wal
04:02:44.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/master-0/data
04:02:44.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
04:02:44.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
04:02:44.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
04:02:44.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:02:44.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:02:44.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:02:44.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/master-0/logs
04:02:44.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/master-0/data/info.pb
04:02:44.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:02:44.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
04:02:44.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
04:02:44.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
04:02:44.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
04:02:44.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:02:44.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:02:44.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:02:44.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
04:02:44.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
04:02:44.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:02:44.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:02:44.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.242.126:44287
04:02:44.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.242.126
04:02:44.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:02:44.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.242.84:45421
04:02:44.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:02:44.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:02:44.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:02:44.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:02:44.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.242.126:44287,127.6.242.125:39089,127.6.242.124:34319
04:02:44.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:02:44.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44651/jwks.json with env {}
04:02:45.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:45.013450 7125 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
04:02:45.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:45.013986 7125 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
04:02:45.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:45.014685 7125 flags.cc:425] Enabled unsafe flag: --never_fsync=true
04:02:45.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:45.045956 7125 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
04:02:45.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:45.046236 7125 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
04:02:45.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:45.046473 7125 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
04:02:45.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:45.046849 7125 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
04:02:45.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:45.047109 7125 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
04:02:45.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:45.047248 7125 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44651/jwks.json
04:02:45.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:45.103572 7125 master_runner.cc:386] Master server non-default flags:
04:02:45.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:02:45.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.242.84:45421
04:02:45.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:02:45.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:02:45.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/master-0/data
04:02:45.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/master-0/wal
04:02:45.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:02:45.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.242.126:44287,127.6.242.125:39089,127.6.242.124:34319
04:02:45.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:02:45.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:02:45.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:02:45.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.242.126:44287
04:02:45.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:02:45.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
04:02:45.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:02:45.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44651/jwks.json
04:02:45.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:02:45.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:02:45.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/master-0/data/info.pb
04:02:45.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.242.126
04:02:45.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:02:45.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
04:02:45.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:02:45.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
04:02:45.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
04:02:45.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:02:45.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/master-0/logs
04:02:45.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:02:45.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
04:02:45.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
04:02:45.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
04:02:45.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
04:02:45.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
04:02:45.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
04:02:45.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 02:50:13 UTC on ccycloud.quasar-purgqw.root.comops.site
04:02:45.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676527
04:02:45.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
04:02:45.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:45.105404 7125 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
04:02:45.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:45.107992 7125 file_cache.cc:493] Constructed file cache file cache with capacity 419430
04:02:45.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:45.126713 7132 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:02:46.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:45.127010 7131 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:02:46.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:45.127461 7125 server_base.cc:1034] running on GCE node
04:02:46.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:45.127942 7134 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:02:46.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:46.303789 7125 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
04:02:46.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:46.306463 7125 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
04:02:46.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:46.307842 7125 hybrid_clock.cc:648] HybridClock initialized: now 1790568166307812 us; error 35 us; skew 500 ppm
04:02:46.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:46.308843 7125 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
04:02:46.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:46.314963 7125 webserver.cc:460] Webserver started at http://127.6.242.126:35835/ using document root <none> and password file <none>
04:02:46.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:46.315933 7125 fs_manager.cc:359] Metadata directory not provided
04:02:46.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:46.316175 7125 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
04:02:46.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:46.316794 7125 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
04:02:46.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:46.321755 7125 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/master-0/data/instance:
04:02:46.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fdd06fdfb82a4d1caf216183316d41d8"
04:02:46.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 04:02:46 on dist-test-slave-jlk3"
04:02:46.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:46.322983 7125 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/master-0/wal/instance:
04:02:46.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fdd06fdfb82a4d1caf216183316d41d8"
04:02:46.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 04:02:46 on dist-test-slave-jlk3"
04:02:46.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:46.329980 7125 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.004s sys 0.004s
04:02:46.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:46.335393 7141 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
04:02:46.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:46.336385 7125 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
04:02:46.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:46.336755 7125 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/master-0/data,/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/master-0/wal
04:02:46.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fdd06fdfb82a4d1caf216183316d41d8"
04:02:46.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 04:02:46 on dist-test-slave-jlk3"
04:02:46.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:46.337124 7125 fs_report.cc:383] FS layout report
04:02:46.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
04:02:46.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/master-0/wal
04:02:46.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/master-0/wal
04:02:46.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/master-0/data/data
04:02:46.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
04:02:46.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
04:02:46.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
04:02:46.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
04:02:46.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
04:02:46.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
04:02:46.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
04:02:46.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
04:02:46.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
04:02:46.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
04:02:46.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
04:02:46.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:46.376259 7125 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
04:02:46.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:46.377863 7125 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
04:02:46.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:46.378415 7125 kserver.cc:163] Server-wide thread pool size limit: 3276
04:02:46.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:46.447588 7125 rpc_server.cc:307] RPC server started. Bound to: 127.6.242.126:44287
04:02:46.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:46.447683 7192 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.242.126:44287 every 8 connection(s)
04:02:46.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:46.450685 7125 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/master-0/data/info.pb
04:02:46.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:46.455925 7193 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
04:02:46.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:46.460315 7113 external_mini_cluster.cc:1388] Started /tmp/dist-test-task4Qe23I/build/tsan/bin/kudu as pid 7125
04:02:46.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:46.460894 7113 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/master-0/wal/instance
04:02:46.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:46.465443 7113 external_mini_cluster.cc:1326] Running /tmp/dist-test-task4Qe23I/build/tsan/bin/kudu
04:02:46.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4Qe23I/build/tsan/bin/kudu
04:02:46.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/master-1/wal
04:02:46.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/master-1/data
04:02:46.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
04:02:46.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
04:02:46.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
04:02:46.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:02:46.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:02:46.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:02:46.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/master-1/logs
04:02:46.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/master-1/data/info.pb
04:02:46.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:02:46.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
04:02:46.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
04:02:46.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
04:02:46.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
04:02:46.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:02:46.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:02:46.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:02:46.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
04:02:46.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
04:02:46.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:02:46.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:02:46.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.242.125:39089
04:02:46.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.242.125
04:02:46.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:02:46.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.242.84:45421
04:02:46.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:02:46.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:02:46.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:02:46.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:02:46.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.242.126:44287,127.6.242.125:39089,127.6.242.124:34319
04:02:46.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:02:46.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44651/jwks.json with env {}
04:02:46.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:46.467571 7193 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.242.126" port: 44287 } has no permanent_uuid. Determining permanent_uuid...
04:02:46.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:46.501905 7193 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.242.125" port: 39089 } has no permanent_uuid. Determining permanent_uuid...
04:02:46.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:46.504278 7143 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.242.125:39089: connect: Connection refused (error 111)
04:02:46.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:46.508031 7193 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.242.125:39089: Network error: Client connection negotiation failed: client connection to 127.6.242.125:39089: connect: Connection refused (error 111)
04:02:46.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:46.557510 7193 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.242.125" port: 39089 } attempt: 1
04:02:46.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:46.561416 7193 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.242.125:39089: Network error: Client connection negotiation failed: client connection to 127.6.242.125:39089: connect: Connection refused (error 111)
04:02:46.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:46.629824 7193 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.242.125" port: 39089 } attempt: 2
04:02:46.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:46.633641 7193 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.242.125:39089: Network error: Client connection negotiation failed: client connection to 127.6.242.125:39089: connect: Connection refused (error 111)
04:02:46.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:46.725037 7193 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.242.125" port: 39089 } attempt: 3
04:02:46.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:46.728762 7193 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.242.125:39089: Network error: Client connection negotiation failed: client connection to 127.6.242.125:39089: connect: Connection refused (error 111)
04:02:46.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:46.770259 7195 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
04:02:46.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:46.770815 7195 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
04:02:46.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:46.771674 7195 flags.cc:425] Enabled unsafe flag: --never_fsync=true
04:02:46.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:46.802758 7195 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
04:02:46.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:46.803063 7195 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
04:02:46.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:46.803242 7195 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
04:02:46.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:46.803412 7195 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
04:02:46.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:46.803682 7195 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
04:02:46.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:46.803943 7195 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44651/jwks.json
04:02:46.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:46.838969 7195 master_runner.cc:386] Master server non-default flags:
04:02:46.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:02:46.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.242.84:45421
04:02:46.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:02:46.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:02:46.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/master-1/data
04:02:46.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/master-1/wal
04:02:46.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:02:46.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.242.126:44287,127.6.242.125:39089,127.6.242.124:34319
04:02:46.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:02:46.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:02:46.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:02:46.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.242.125:39089
04:02:46.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:02:46.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
04:02:46.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:02:46.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44651/jwks.json
04:02:46.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:02:46.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:02:46.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/master-1/data/info.pb
04:02:46.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.242.125
04:02:46.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:02:46.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
04:02:46.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:02:46.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
04:02:46.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
04:02:46.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:02:46.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/master-1/logs
04:02:46.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:02:46.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
04:02:46.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
04:02:46.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
04:02:46.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
04:02:46.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
04:02:46.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
04:02:46.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 02:50:13 UTC on ccycloud.quasar-purgqw.root.comops.site
04:02:46.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676527
04:02:46.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
04:02:46.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:46.840266 7195 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
04:02:46.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:46.842428 7195 file_cache.cc:493] Constructed file cache file cache with capacity 419430
04:02:46.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:46.860857 7204 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:02:46.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:46.872143 7193 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.242.125" port: 39089 } attempt: 4
04:02:46.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:46.876101 7193 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.242.125:39089: Network error: Client connection negotiation failed: client connection to 127.6.242.125:39089: connect: Connection refused (error 111)
04:02:47.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:47.175534 7193 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.242.125" port: 39089 } attempt: 5
04:02:47.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:47.180099 7193 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.242.125:39089: Network error: Client connection negotiation failed: client connection to 127.6.242.125:39089: connect: Connection refused (error 111)
04:02:47.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:47.727527 7193 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.242.125" port: 39089 } attempt: 6
04:02:47.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:47.734673 7193 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.242.125:39089: Network error: Client connection negotiation failed: client connection to 127.6.242.125:39089: connect: Connection refused (error 111)
04:02:48.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:46.860881 7205 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:02:48.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:46.860898 7207 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:02:48.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:46.862289 7195 server_base.cc:1034] running on GCE node
04:02:48.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:48.054941 7195 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
04:02:48.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:48.057928 7195 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
04:02:48.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:48.059326 7195 hybrid_clock.cc:648] HybridClock initialized: now 1790568168059280 us; error 76 us; skew 500 ppm
04:02:48.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:48.060514 7195 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
04:02:48.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:48.067804 7195 webserver.cc:460] Webserver started at http://127.6.242.125:35807/ using document root <none> and password file <none>
04:02:48.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:48.068971 7195 fs_manager.cc:359] Metadata directory not provided
04:02:48.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:48.069243 7195 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
04:02:48.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:48.069842 7195 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
04:02:48.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:48.075709 7195 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/master-1/data/instance:
04:02:48.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f21e4c0341ef4a30bb9d8f056b478ec8"
04:02:48.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 04:02:48 on dist-test-slave-jlk3"
04:02:48.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:48.077023 7195 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/master-1/wal/instance:
04:02:48.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f21e4c0341ef4a30bb9d8f056b478ec8"
04:02:48.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 04:02:48 on dist-test-slave-jlk3"
04:02:48.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:48.084327 7195 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.000s
04:02:48.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:48.089866 7216 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
04:02:48.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:48.090886 7195 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
04:02:48.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:48.091238 7195 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/master-1/data,/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/master-1/wal
04:02:48.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f21e4c0341ef4a30bb9d8f056b478ec8"
04:02:48.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 04:02:48 on dist-test-slave-jlk3"
04:02:48.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:48.091624 7195 fs_report.cc:383] FS layout report
04:02:48.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
04:02:48.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/master-1/wal
04:02:48.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/master-1/wal
04:02:48.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/master-1/data/data
04:02:48.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
04:02:48.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
04:02:48.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
04:02:48.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
04:02:48.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
04:02:48.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
04:02:48.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
04:02:48.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
04:02:48.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
04:02:48.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
04:02:48.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
04:02:48.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:48.135946 7195 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
04:02:48.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:48.137547 7195 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
04:02:48.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:48.138055 7195 kserver.cc:163] Server-wide thread pool size limit: 3276
04:02:48.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:48.206511 7195 rpc_server.cc:307] RPC server started. Bound to: 127.6.242.125:39089
04:02:48.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:48.206612 7267 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.242.125:39089 every 8 connection(s)
04:02:48.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:48.209277 7195 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/master-1/data/info.pb
04:02:48.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:48.210624 7113 external_mini_cluster.cc:1388] Started /tmp/dist-test-task4Qe23I/build/tsan/bin/kudu as pid 7195
04:02:48.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:48.211143 7113 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/master-1/wal/instance
04:02:48.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:48.214205 7113 external_mini_cluster.cc:1326] Running /tmp/dist-test-task4Qe23I/build/tsan/bin/kudu
04:02:48.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4Qe23I/build/tsan/bin/kudu
04:02:48.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/master-2/wal
04:02:48.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/master-2/data
04:02:48.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
04:02:48.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
04:02:48.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
04:02:48.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:02:48.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:02:48.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:02:48.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/master-2/logs
04:02:48.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/master-2/data/info.pb
04:02:48.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:02:48.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
04:02:48.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
04:02:48.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
04:02:48.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
04:02:48.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:02:48.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:02:48.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:02:48.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
04:02:48.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
04:02:48.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:02:48.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:02:48.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.242.124:34319
04:02:48.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.242.124
04:02:48.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:02:48.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.242.84:45421
04:02:48.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:02:48.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:02:48.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:02:48.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:02:48.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.242.126:44287,127.6.242.125:39089,127.6.242.124:34319
04:02:48.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:02:48.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44651/jwks.json with env {}
04:02:48.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:48.215799 7268 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
04:02:48.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:48.230615 7268 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.242.126" port: 44287 } has no permanent_uuid. Determining permanent_uuid...
04:02:48.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:48.253043 7268 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.242.125" port: 39089 } has no permanent_uuid. Determining permanent_uuid...
04:02:48.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:48.268838 7268 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.242.124" port: 34319 } has no permanent_uuid. Determining permanent_uuid...
04:02:48.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:48.271013 7218 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.242.124:34319: connect: Connection refused (error 111)
04:02:48.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:48.275640 7268 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.242.124:34319: Network error: Client connection negotiation failed: client connection to 127.6.242.124:34319: connect: Connection refused (error 111)
04:02:48.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:48.325140 7268 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.242.124" port: 34319 } attempt: 1
04:02:48.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:48.328879 7268 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.242.124:34319: Network error: Client connection negotiation failed: client connection to 127.6.242.124:34319: connect: Connection refused (error 111)
04:02:48.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:48.397365 7268 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.242.124" port: 34319 } attempt: 2
04:02:48.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:48.401340 7268 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.242.124:34319: Network error: Client connection negotiation failed: client connection to 127.6.242.124:34319: connect: Connection refused (error 111)
04:02:48.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:48.492789 7268 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.242.124" port: 34319 } attempt: 3
04:02:48.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:48.496488 7268 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.242.124:34319: Network error: Client connection negotiation failed: client connection to 127.6.242.124:34319: connect: Connection refused (error 111)
04:02:48.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:48.517637 7270 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
04:02:48.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:48.518191 7270 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
04:02:48.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:48.518828 7270 flags.cc:425] Enabled unsafe flag: --never_fsync=true
04:02:48.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:48.549937 7270 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
04:02:48.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:48.550236 7270 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
04:02:48.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:48.550427 7270 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
04:02:48.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:48.550602 7270 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
04:02:48.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:48.550904 7270 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
04:02:48.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:48.551133 7270 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44651/jwks.json
04:02:48.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:48.585675 7270 master_runner.cc:386] Master server non-default flags:
04:02:48.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:02:48.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.242.84:45421
04:02:48.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:02:48.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:02:48.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/master-2/data
04:02:48.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/master-2/wal
04:02:48.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:02:48.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.242.126:44287,127.6.242.125:39089,127.6.242.124:34319
04:02:48.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:02:48.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:02:48.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:02:48.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.242.124:34319
04:02:48.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:02:48.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
04:02:48.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:02:48.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44651/jwks.json
04:02:48.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:02:48.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:02:48.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/master-2/data/info.pb
04:02:48.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.242.124
04:02:48.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:02:48.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
04:02:48.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:02:48.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
04:02:48.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
04:02:48.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:02:48.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/master-2/logs
04:02:48.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:02:48.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
04:02:48.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
04:02:48.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
04:02:48.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
04:02:48.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
04:02:48.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
04:02:48.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 02:50:13 UTC on ccycloud.quasar-purgqw.root.comops.site
04:02:48.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676527
04:02:48.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
04:02:48.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:48.587077 7270 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
04:02:48.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:48.589110 7270 file_cache.cc:493] Constructed file cache file cache with capacity 419430
04:02:48.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:48.608614 7280 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:02:48.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:48.640002 7268 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.242.124" port: 34319 } attempt: 4
04:02:48.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:48.643937 7268 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.242.124:34319: Network error: Client connection negotiation failed: client connection to 127.6.242.124:34319: connect: Connection refused (error 111)
04:02:48.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:48.795296 7193 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.242.125" port: 39089 } attempt: 7
04:02:48.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:48.818445 7193 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.242.124" port: 34319 } has no permanent_uuid. Determining permanent_uuid...
04:02:48.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:48.823405 7193 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.242.124:34319: Network error: Client connection negotiation failed: client connection to 127.6.242.124:34319: connect: Connection refused (error 111)
04:02:48.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:48.882001 7193 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.242.124" port: 34319 } attempt: 1
04:02:48.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:48.886165 7193 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.242.124:34319: Network error: Client connection negotiation failed: client connection to 127.6.242.124:34319: connect: Connection refused (error 111)
04:02:48.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:48.943598 7268 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.242.124" port: 34319 } attempt: 5
04:02:48.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:48.947732 7268 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.242.124:34319: Network error: Client connection negotiation failed: client connection to 127.6.242.124:34319: connect: Connection refused (error 111)
04:02:48.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:48.967566 7193 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.242.124" port: 34319 } attempt: 2
04:02:48.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:48.971977 7193 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.242.124:34319: Network error: Client connection negotiation failed: client connection to 127.6.242.124:34319: connect: Connection refused (error 111)
04:02:49.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:49.057376 7193 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.242.124" port: 34319 } attempt: 3
04:02:49.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:49.061691 7193 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.242.124:34319: Network error: Client connection negotiation failed: client connection to 127.6.242.124:34319: connect: Connection refused (error 111)
04:02:49.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:49.202078 7193 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.242.124" port: 34319 } attempt: 4
04:02:49.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:49.210582 7193 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.242.124:34319: Network error: Client connection negotiation failed: client connection to 127.6.242.124:34319: connect: Connection refused (error 111)
04:02:49.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:49.494014 7193 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.242.124" port: 34319 } attempt: 5
04:02:49.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:49.495337 7268 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.242.124" port: 34319 } attempt: 6
04:02:49.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:49.499687 7193 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.242.124:34319: Network error: Client connection negotiation failed: client connection to 127.6.242.124:34319: connect: Connection refused (error 111)
04:02:49.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:49.504127 7268 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.242.124:34319: Network error: Client connection negotiation failed: client connection to 127.6.242.124:34319: connect: Connection refused (error 111)
04:02:49.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:48.608613 7281 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:02:49.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:48.609059 7283 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:02:49.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:48.610435 7270 server_base.cc:1034] running on GCE node
04:02:49.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:49.804989 7270 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
04:02:49.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:49.807633 7270 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
04:02:49.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:49.808984 7270 hybrid_clock.cc:648] HybridClock initialized: now 1790568169808954 us; error 51 us; skew 500 ppm
04:02:49.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:49.809780 7270 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
04:02:49.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:49.816047 7270 webserver.cc:460] Webserver started at http://127.6.242.124:41567/ using document root <none> and password file <none>
04:02:49.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:49.817080 7270 fs_manager.cc:359] Metadata directory not provided
04:02:49.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:49.817263 7270 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
04:02:49.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:49.817631 7270 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
04:02:49.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:49.824456 7270 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/master-2/data/instance:
04:02:49.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cb0d04479eb54032b790cb51a14b778e"
04:02:49.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 04:02:49 on dist-test-slave-jlk3"
04:02:49.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:49.825992 7270 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/master-2/wal/instance:
04:02:49.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cb0d04479eb54032b790cb51a14b778e"
04:02:49.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 04:02:49 on dist-test-slave-jlk3"
04:02:49.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:49.835180 7270 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.009s sys 0.000s
04:02:49.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:49.842388 7295 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
04:02:49.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:49.843595 7270 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
04:02:49.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:49.843875 7270 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/master-2/data,/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/master-2/wal
04:02:49.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cb0d04479eb54032b790cb51a14b778e"
04:02:49.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 04:02:49 on dist-test-slave-jlk3"
04:02:49.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:49.844190 7270 fs_report.cc:383] FS layout report
04:02:49.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
04:02:49.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/master-2/wal
04:02:49.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/master-2/wal
04:02:49.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/master-2/data/data
04:02:49.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
04:02:49.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
04:02:49.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
04:02:49.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
04:02:49.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
04:02:49.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
04:02:49.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
04:02:49.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
04:02:49.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
04:02:49.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
04:02:49.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
04:02:49.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:49.880992 7270 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
04:02:49.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:49.882383 7270 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
04:02:49.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:49.882769 7270 kserver.cc:163] Server-wide thread pool size limit: 3276
04:02:49.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:49.951749 7270 rpc_server.cc:307] RPC server started. Bound to: 127.6.242.124:34319
04:02:49.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:49.951864 7346 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.242.124:34319 every 8 connection(s)
04:02:49.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:49.954488 7270 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/master-2/data/info.pb
04:02:49.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:49.959344 7113 external_mini_cluster.cc:1388] Started /tmp/dist-test-task4Qe23I/build/tsan/bin/kudu as pid 7270
04:02:49.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:49.959770 7113 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/master-2/wal/instance
04:02:49.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:49.961227 7347 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
04:02:49.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:49.976846 7347 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.242.126" port: 44287 } has no permanent_uuid. Determining permanent_uuid...
04:02:50.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.013695 7347 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.242.125" port: 39089 } has no permanent_uuid. Determining permanent_uuid...
04:02:50.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.025228 7347 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.242.124" port: 34319 } has no permanent_uuid. Determining permanent_uuid...
04:02:50.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.042551 7347 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P cb0d04479eb54032b790cb51a14b778e: Bootstrap starting.
04:02:50.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.048663 7347 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P cb0d04479eb54032b790cb51a14b778e: Neither blocks nor log segments found. Creating new log.
04:02:50.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.050473 7347 log.cc:824] T 00000000000000000000000000000000 P cb0d04479eb54032b790cb51a14b778e: Log is configured to *not* fsync() on all Append() calls
04:02:50.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.052242 7193 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.242.124" port: 34319 } attempt: 6
04:02:50.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.055238 7347 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P cb0d04479eb54032b790cb51a14b778e: No bootstrap required, opened a new log
04:02:50.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.080125 7193 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P fdd06fdfb82a4d1caf216183316d41d8: Bootstrap starting.
04:02:50.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.082837 7347 raft_consensus.cc:348] T 00000000000000000000000000000000 P cb0d04479eb54032b790cb51a14b778e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fdd06fdfb82a4d1caf216183316d41d8" member_type: VOTER last_known_addr { host: "127.6.242.126" port: 44287 } } peers { permanent_uuid: "f21e4c0341ef4a30bb9d8f056b478ec8" member_type: VOTER last_known_addr { host: "127.6.242.125" port: 39089 } } peers { permanent_uuid: "cb0d04479eb54032b790cb51a14b778e" member_type: VOTER last_known_addr { host: "127.6.242.124" port: 34319 } }
04:02:50.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.084084 7347 raft_consensus.cc:374] T 00000000000000000000000000000000 P cb0d04479eb54032b790cb51a14b778e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
04:02:50.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.084635 7347 raft_consensus.cc:729] T 00000000000000000000000000000000 P cb0d04479eb54032b790cb51a14b778e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: cb0d04479eb54032b790cb51a14b778e, State: Initialized, Role: FOLLOWER
04:02:50.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.085839 7347 consensus_queue.cc:260] T 00000000000000000000000000000000 P cb0d04479eb54032b790cb51a14b778e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fdd06fdfb82a4d1caf216183316d41d8" member_type: VOTER last_known_addr { host: "127.6.242.126" port: 44287 } } peers { permanent_uuid: "f21e4c0341ef4a30bb9d8f056b478ec8" member_type: VOTER last_known_addr { host: "127.6.242.125" port: 39089 } } peers { permanent_uuid: "cb0d04479eb54032b790cb51a14b778e" member_type: VOTER last_known_addr { host: "127.6.242.124" port: 34319 } }
04:02:50.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.087239 7193 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P fdd06fdfb82a4d1caf216183316d41d8: Neither blocks nor log segments found. Creating new log.
04:02:50.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.088660 7357 sys_catalog.cc:455] T 00000000000000000000000000000000 P cb0d04479eb54032b790cb51a14b778e [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fdd06fdfb82a4d1caf216183316d41d8" member_type: VOTER last_known_addr { host: "127.6.242.126" port: 44287 } } peers { permanent_uuid: "f21e4c0341ef4a30bb9d8f056b478ec8" member_type: VOTER last_known_addr { host: "127.6.242.125" port: 39089 } } peers { permanent_uuid: "cb0d04479eb54032b790cb51a14b778e" member_type: VOTER last_known_addr { host: "127.6.242.124" port: 34319 } } }
04:02:50.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.089363 7357 sys_catalog.cc:458] T 00000000000000000000000000000000 P cb0d04479eb54032b790cb51a14b778e [sys.catalog]: This master's current role is: FOLLOWER
04:02:50.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.089692 7193 log.cc:824] T 00000000000000000000000000000000 P fdd06fdfb82a4d1caf216183316d41d8: Log is configured to *not* fsync() on all Append() calls
04:02:50.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.090660 7347 sys_catalog.cc:564] T 00000000000000000000000000000000 P cb0d04479eb54032b790cb51a14b778e [sys.catalog]: configured and running, proceeding with master startup.
04:02:50.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.095156 7193 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P fdd06fdfb82a4d1caf216183316d41d8: No bootstrap required, opened a new log
04:02:50.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:50.123121 7369 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cb0d04479eb54032b790cb51a14b778e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:02:50.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:50.124572 7369 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:02:50.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.128373 7193 raft_consensus.cc:348] T 00000000000000000000000000000000 P fdd06fdfb82a4d1caf216183316d41d8 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fdd06fdfb82a4d1caf216183316d41d8" member_type: VOTER last_known_addr { host: "127.6.242.126" port: 44287 } } peers { permanent_uuid: "f21e4c0341ef4a30bb9d8f056b478ec8" member_type: VOTER last_known_addr { host: "127.6.242.125" port: 39089 } } peers { permanent_uuid: "cb0d04479eb54032b790cb51a14b778e" member_type: VOTER last_known_addr { host: "127.6.242.124" port: 34319 } }
04:02:50.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.129218 7193 raft_consensus.cc:374] T 00000000000000000000000000000000 P fdd06fdfb82a4d1caf216183316d41d8 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
04:02:50.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.129490 7193 raft_consensus.cc:729] T 00000000000000000000000000000000 P fdd06fdfb82a4d1caf216183316d41d8 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fdd06fdfb82a4d1caf216183316d41d8, State: Initialized, Role: FOLLOWER
04:02:50.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.130638 7193 consensus_queue.cc:260] T 00000000000000000000000000000000 P fdd06fdfb82a4d1caf216183316d41d8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fdd06fdfb82a4d1caf216183316d41d8" member_type: VOTER last_known_addr { host: "127.6.242.126" port: 44287 } } peers { permanent_uuid: "f21e4c0341ef4a30bb9d8f056b478ec8" member_type: VOTER last_known_addr { host: "127.6.242.125" port: 39089 } } peers { permanent_uuid: "cb0d04479eb54032b790cb51a14b778e" member_type: VOTER last_known_addr { host: "127.6.242.124" port: 34319 } }
04:02:50.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.136947 7371 sys_catalog.cc:455] T 00000000000000000000000000000000 P fdd06fdfb82a4d1caf216183316d41d8 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fdd06fdfb82a4d1caf216183316d41d8" member_type: VOTER last_known_addr { host: "127.6.242.126" port: 44287 } } peers { permanent_uuid: "f21e4c0341ef4a30bb9d8f056b478ec8" member_type: VOTER last_known_addr { host: "127.6.242.125" port: 39089 } } peers { permanent_uuid: "cb0d04479eb54032b790cb51a14b778e" member_type: VOTER last_known_addr { host: "127.6.242.124" port: 34319 } } }
04:02:50.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:50.137734 7301 tablet.cc:2369] T 00000000000000000000000000000000 P cb0d04479eb54032b790cb51a14b778e: Can't schedule compaction. Clean time has not been advanced past its initial value.
04:02:50.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:50.137847 7147 tablet.cc:2369] T 00000000000000000000000000000000 P fdd06fdfb82a4d1caf216183316d41d8: Can't schedule compaction. Clean time has not been advanced past its initial value.
04:02:50.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.139310 7193 sys_catalog.cc:564] T 00000000000000000000000000000000 P fdd06fdfb82a4d1caf216183316d41d8 [sys.catalog]: configured and running, proceeding with master startup.
04:02:50.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.139569 7371 sys_catalog.cc:458] T 00000000000000000000000000000000 P fdd06fdfb82a4d1caf216183316d41d8 [sys.catalog]: This master's current role is: FOLLOWER
04:02:50.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.150000 7371 raft_consensus.cc:482] T 00000000000000000000000000000000 P fdd06fdfb82a4d1caf216183316d41d8 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
04:02:50.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.150475 7371 raft_consensus.cc:504] T 00000000000000000000000000000000 P fdd06fdfb82a4d1caf216183316d41d8 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fdd06fdfb82a4d1caf216183316d41d8" member_type: VOTER last_known_addr { host: "127.6.242.126" port: 44287 } } peers { permanent_uuid: "f21e4c0341ef4a30bb9d8f056b478ec8" member_type: VOTER last_known_addr { host: "127.6.242.125" port: 39089 } } peers { permanent_uuid: "cb0d04479eb54032b790cb51a14b778e" member_type: VOTER last_known_addr { host: "127.6.242.124" port: 34319 } }
04:02:50.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.156055 7371 leader_election.cc:288] T 00000000000000000000000000000000 P fdd06fdfb82a4d1caf216183316d41d8 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f21e4c0341ef4a30bb9d8f056b478ec8 (127.6.242.125:39089), cb0d04479eb54032b790cb51a14b778e (127.6.242.124:34319)
04:02:50.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.158066 7322 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "fdd06fdfb82a4d1caf216183316d41d8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "cb0d04479eb54032b790cb51a14b778e" is_pre_election: true
04:02:50.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.158951 7322 raft_consensus.cc:2454] T 00000000000000000000000000000000 P cb0d04479eb54032b790cb51a14b778e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate fdd06fdfb82a4d1caf216183316d41d8 in term 0.
04:02:50.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.160382 7143 leader_election.cc:302] T 00000000000000000000000000000000 P fdd06fdfb82a4d1caf216183316d41d8 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: cb0d04479eb54032b790cb51a14b778e, fdd06fdfb82a4d1caf216183316d41d8; no voters:
04:02:50.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.120579 7270 master_runner.cc:186] Error getting master registration for 127.6.242.126:44287: OK, instance_id { permanent_uuid: "fdd06fdfb82a4d1caf216183316d41d8" instance_seqno: 1790568166428831 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
04:02:50.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.161986 7270 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
04:02:50.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.163518 7371 raft_consensus.cc:2781] T 00000000000000000000000000000000 P fdd06fdfb82a4d1caf216183316d41d8 [term 0 FOLLOWER]: Leader pre-election won for term 1
04:02:50.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.164654 7371 raft_consensus.cc:482] T 00000000000000000000000000000000 P fdd06fdfb82a4d1caf216183316d41d8 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
04:02:50.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.165855 7371 raft_consensus.cc:3037] T 00000000000000000000000000000000 P fdd06fdfb82a4d1caf216183316d41d8 [term 0 FOLLOWER]: Advancing to term 1
04:02:50.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.186385 7371 raft_consensus.cc:504] T 00000000000000000000000000000000 P fdd06fdfb82a4d1caf216183316d41d8 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fdd06fdfb82a4d1caf216183316d41d8" member_type: VOTER last_known_addr { host: "127.6.242.126" port: 44287 } } peers { permanent_uuid: "f21e4c0341ef4a30bb9d8f056b478ec8" member_type: VOTER last_known_addr { host: "127.6.242.125" port: 39089 } } peers { permanent_uuid: "cb0d04479eb54032b790cb51a14b778e" member_type: VOTER last_known_addr { host: "127.6.242.124" port: 34319 } }
04:02:50.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.188547 7371 leader_election.cc:288] T 00000000000000000000000000000000 P fdd06fdfb82a4d1caf216183316d41d8 [CANDIDATE]: Term 1 election: Requested vote from peers f21e4c0341ef4a30bb9d8f056b478ec8 (127.6.242.125:39089), cb0d04479eb54032b790cb51a14b778e (127.6.242.124:34319)
04:02:50.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.190243 7322 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "fdd06fdfb82a4d1caf216183316d41d8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "cb0d04479eb54032b790cb51a14b778e"
04:02:50.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.190860 7322 raft_consensus.cc:3037] T 00000000000000000000000000000000 P cb0d04479eb54032b790cb51a14b778e [term 0 FOLLOWER]: Advancing to term 1
04:02:50.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.199844 7383 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
04:02:50.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.199970 7322 raft_consensus.cc:2454] T 00000000000000000000000000000000 P cb0d04479eb54032b790cb51a14b778e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate fdd06fdfb82a4d1caf216183316d41d8 in term 1.
04:02:50.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.201314 7143 leader_election.cc:302] T 00000000000000000000000000000000 P fdd06fdfb82a4d1caf216183316d41d8 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: cb0d04479eb54032b790cb51a14b778e, fdd06fdfb82a4d1caf216183316d41d8; no voters:
04:02:50.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.225888 7371 raft_consensus.cc:2781] T 00000000000000000000000000000000 P fdd06fdfb82a4d1caf216183316d41d8 [term 1 FOLLOWER]: Leader election won for term 1
04:02:50.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.238620 7371 raft_consensus.cc:686] T 00000000000000000000000000000000 P fdd06fdfb82a4d1caf216183316d41d8 [term 1 LEADER]: Becoming Leader. State: Replica: fdd06fdfb82a4d1caf216183316d41d8, State: Running, Role: LEADER
04:02:50.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.241997 7371 consensus_queue.cc:237] T 00000000000000000000000000000000 P fdd06fdfb82a4d1caf216183316d41d8 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fdd06fdfb82a4d1caf216183316d41d8" member_type: VOTER last_known_addr { host: "127.6.242.126" port: 44287 } } peers { permanent_uuid: "f21e4c0341ef4a30bb9d8f056b478ec8" member_type: VOTER last_known_addr { host: "127.6.242.125" port: 39089 } } peers { permanent_uuid: "cb0d04479eb54032b790cb51a14b778e" member_type: VOTER last_known_addr { host: "127.6.242.124" port: 34319 } }
04:02:50.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.207819 7125 master_runner.cc:186] Error getting master registration for 127.6.242.125:39089: OK, instance_id { permanent_uuid: "f21e4c0341ef4a30bb9d8f056b478ec8" instance_seqno: 1790568168187299 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
04:02:50.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.248992 7125 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
04:02:50.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.257261 7379 sys_catalog.cc:455] T 00000000000000000000000000000000 P fdd06fdfb82a4d1caf216183316d41d8 [sys.catalog]: SysCatalogTable state changed. Reason: New leader fdd06fdfb82a4d1caf216183316d41d8. Latest consensus state: current_term: 1 leader_uuid: "fdd06fdfb82a4d1caf216183316d41d8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fdd06fdfb82a4d1caf216183316d41d8" member_type: VOTER last_known_addr { host: "127.6.242.126" port: 44287 } } peers { permanent_uuid: "f21e4c0341ef4a30bb9d8f056b478ec8" member_type: VOTER last_known_addr { host: "127.6.242.125" port: 39089 } } peers { permanent_uuid: "cb0d04479eb54032b790cb51a14b778e" member_type: VOTER last_known_addr { host: "127.6.242.124" port: 34319 } } }
04:02:50.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.258033 7379 sys_catalog.cc:458] T 00000000000000000000000000000000 P fdd06fdfb82a4d1caf216183316d41d8 [sys.catalog]: This master's current role is: LEADER
04:02:50.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.262290 7385 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
04:02:50.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.273155 7385 catalog_manager.cc:1484] Initializing Kudu cluster ID...
04:02:50.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.301833 7322 raft_consensus.cc:1261] T 00000000000000000000000000000000 P cb0d04479eb54032b790cb51a14b778e [term 1 FOLLOWER]: Refusing update from remote peer fdd06fdfb82a4d1caf216183316d41d8: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
04:02:50.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.303946 7379 consensus_queue.cc:1035] T 00000000000000000000000000000000 P fdd06fdfb82a4d1caf216183316d41d8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "cb0d04479eb54032b790cb51a14b778e" member_type: VOTER last_known_addr { host: "127.6.242.124" port: 34319 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
04:02:50.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.330636 7357 sys_catalog.cc:455] T 00000000000000000000000000000000 P cb0d04479eb54032b790cb51a14b778e [sys.catalog]: SysCatalogTable state changed. Reason: New leader fdd06fdfb82a4d1caf216183316d41d8. Latest consensus state: current_term: 1 leader_uuid: "fdd06fdfb82a4d1caf216183316d41d8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fdd06fdfb82a4d1caf216183316d41d8" member_type: VOTER last_known_addr { host: "127.6.242.126" port: 44287 } } peers { permanent_uuid: "f21e4c0341ef4a30bb9d8f056b478ec8" member_type: VOTER last_known_addr { host: "127.6.242.125" port: 39089 } } peers { permanent_uuid: "cb0d04479eb54032b790cb51a14b778e" member_type: VOTER last_known_addr { host: "127.6.242.124" port: 34319 } } }
04:02:50.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.331503 7357 sys_catalog.cc:458] T 00000000000000000000000000000000 P cb0d04479eb54032b790cb51a14b778e [sys.catalog]: This master's current role is: FOLLOWER
04:02:50.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.337379 7387 mvcc.cc:205] Tried to move back new op lower bound from 7334167225537392640 to 7334167225344454656. Current Snapshot: MvccSnapshot[applied={T|T < 7334167225537392640}]
04:02:50.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.340309 7388 mvcc.cc:205] Tried to move back new op lower bound from 7334167225537392640 to 7334167225344454656. Current Snapshot: MvccSnapshot[applied={T|T < 7334167225537392640}]
04:02:50.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.350432 7357 sys_catalog.cc:455] T 00000000000000000000000000000000 P cb0d04479eb54032b790cb51a14b778e [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "fdd06fdfb82a4d1caf216183316d41d8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fdd06fdfb82a4d1caf216183316d41d8" member_type: VOTER last_known_addr { host: "127.6.242.126" port: 44287 } } peers { permanent_uuid: "f21e4c0341ef4a30bb9d8f056b478ec8" member_type: VOTER last_known_addr { host: "127.6.242.125" port: 39089 } } peers { permanent_uuid: "cb0d04479eb54032b790cb51a14b778e" member_type: VOTER last_known_addr { host: "127.6.242.124" port: 34319 } } }
04:02:50.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.351166 7357 sys_catalog.cc:458] T 00000000000000000000000000000000 P cb0d04479eb54032b790cb51a14b778e [sys.catalog]: This master's current role is: FOLLOWER
04:02:50.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.359881 7379 sys_catalog.cc:455] T 00000000000000000000000000000000 P fdd06fdfb82a4d1caf216183316d41d8 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "fdd06fdfb82a4d1caf216183316d41d8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fdd06fdfb82a4d1caf216183316d41d8" member_type: VOTER last_known_addr { host: "127.6.242.126" port: 44287 } } peers { permanent_uuid: "f21e4c0341ef4a30bb9d8f056b478ec8" member_type: VOTER last_known_addr { host: "127.6.242.125" port: 39089 } } peers { permanent_uuid: "cb0d04479eb54032b790cb51a14b778e" member_type: VOTER last_known_addr { host: "127.6.242.124" port: 34319 } } }
04:02:50.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.360601 7379 sys_catalog.cc:458] T 00000000000000000000000000000000 P fdd06fdfb82a4d1caf216183316d41d8 [sys.catalog]: This master's current role is: LEADER
04:02:50.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.362658 7385 catalog_manager.cc:1347] Generated new cluster ID: 2830635cd2194ae4829a4bb28c220f5d
04:02:50.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.362957 7385 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
04:02:50.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.418771 7385 catalog_manager.cc:1370] Generated new certificate authority record
04:02:50.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.421396 7385 catalog_manager.cc:1504] Loading token signing keys...
04:02:50.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.452328 7385 catalog_manager.cc:5868] T 00000000000000000000000000000000 P fdd06fdfb82a4d1caf216183316d41d8: Generated new TSK 0
04:02:50.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.453866 7385 catalog_manager.cc:1514] Initializing in-progress tserver states...
04:02:50.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.479985 7113 external_mini_cluster.cc:1326] Running /tmp/dist-test-task4Qe23I/build/tsan/bin/kudu
04:02:50.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4Qe23I/build/tsan/bin/kudu
04:02:50.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/ts-0/wal
04:02:50.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/ts-0/data
04:02:50.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
04:02:50.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
04:02:50.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
04:02:50.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:02:50.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:02:50.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:02:50.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/ts-0/logs
04:02:50.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/ts-0/data/info.pb
04:02:50.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:02:50.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
04:02:50.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
04:02:50.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
04:02:50.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
04:02:50.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:02:50.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:02:50.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:02:50.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
04:02:50.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
04:02:50.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.242.65:0
04:02:50.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.242.65
04:02:50.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.242.65
04:02:50.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:02:50.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.242.126:44287,127.6.242.125:39089,127.6.242.124:34319
04:02:50.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.242.84:45421
04:02:50.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:02:50.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:02:50.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
04:02:50.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.564523 7268 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.242.124" port: 34319 } attempt: 7
04:02:50.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.604938 7268 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f21e4c0341ef4a30bb9d8f056b478ec8: Bootstrap starting.
04:02:50.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.623848 7268 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P f21e4c0341ef4a30bb9d8f056b478ec8: Neither blocks nor log segments found. Creating new log.
04:02:50.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.629469 7268 log.cc:824] T 00000000000000000000000000000000 P f21e4c0341ef4a30bb9d8f056b478ec8: Log is configured to *not* fsync() on all Append() calls
04:02:50.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.652486 7268 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f21e4c0341ef4a30bb9d8f056b478ec8: No bootstrap required, opened a new log
04:02:50.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.730499 7268 raft_consensus.cc:348] T 00000000000000000000000000000000 P f21e4c0341ef4a30bb9d8f056b478ec8 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fdd06fdfb82a4d1caf216183316d41d8" member_type: VOTER last_known_addr { host: "127.6.242.126" port: 44287 } } peers { permanent_uuid: "f21e4c0341ef4a30bb9d8f056b478ec8" member_type: VOTER last_known_addr { host: "127.6.242.125" port: 39089 } } peers { permanent_uuid: "cb0d04479eb54032b790cb51a14b778e" member_type: VOTER last_known_addr { host: "127.6.242.124" port: 34319 } }
04:02:50.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.732548 7268 raft_consensus.cc:374] T 00000000000000000000000000000000 P f21e4c0341ef4a30bb9d8f056b478ec8 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
04:02:50.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.733359 7268 raft_consensus.cc:729] T 00000000000000000000000000000000 P f21e4c0341ef4a30bb9d8f056b478ec8 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f21e4c0341ef4a30bb9d8f056b478ec8, State: Initialized, Role: FOLLOWER
04:02:50.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.735554 7268 consensus_queue.cc:260] T 00000000000000000000000000000000 P f21e4c0341ef4a30bb9d8f056b478ec8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fdd06fdfb82a4d1caf216183316d41d8" member_type: VOTER last_known_addr { host: "127.6.242.126" port: 44287 } } peers { permanent_uuid: "f21e4c0341ef4a30bb9d8f056b478ec8" member_type: VOTER last_known_addr { host: "127.6.242.125" port: 39089 } } peers { permanent_uuid: "cb0d04479eb54032b790cb51a14b778e" member_type: VOTER last_known_addr { host: "127.6.242.124" port: 34319 } }
04:02:50.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.758301 7242 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "fdd06fdfb82a4d1caf216183316d41d8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f21e4c0341ef4a30bb9d8f056b478ec8"
04:02:50.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.759575 7242 raft_consensus.cc:3037] T 00000000000000000000000000000000 P f21e4c0341ef4a30bb9d8f056b478ec8 [term 0 FOLLOWER]: Advancing to term 1
04:02:50.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.760619 7243 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "fdd06fdfb82a4d1caf216183316d41d8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f21e4c0341ef4a30bb9d8f056b478ec8" is_pre_election: true
04:02:50.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.767935 7242 raft_consensus.cc:2454] T 00000000000000000000000000000000 P f21e4c0341ef4a30bb9d8f056b478ec8 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate fdd06fdfb82a4d1caf216183316d41d8 in term 1.
04:02:50.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.769313 7241 raft_consensus.cc:1261] T 00000000000000000000000000000000 P f21e4c0341ef4a30bb9d8f056b478ec8 [term 1 FOLLOWER]: Refusing update from remote peer fdd06fdfb82a4d1caf216183316d41d8: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
04:02:50.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.775776 7371 consensus_queue.cc:1035] T 00000000000000000000000000000000 P fdd06fdfb82a4d1caf216183316d41d8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f21e4c0341ef4a30bb9d8f056b478ec8" member_type: VOTER last_known_addr { host: "127.6.242.125" port: 39089 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
04:02:50.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.758068 7268 sys_catalog.cc:564] T 00000000000000000000000000000000 P f21e4c0341ef4a30bb9d8f056b478ec8 [sys.catalog]: configured and running, proceeding with master startup.
04:02:50.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.777047 7395 sys_catalog.cc:455] T 00000000000000000000000000000000 P f21e4c0341ef4a30bb9d8f056b478ec8 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fdd06fdfb82a4d1caf216183316d41d8" member_type: VOTER last_known_addr { host: "127.6.242.126" port: 44287 } } peers { permanent_uuid: "f21e4c0341ef4a30bb9d8f056b478ec8" member_type: VOTER last_known_addr { host: "127.6.242.125" port: 39089 } } peers { permanent_uuid: "cb0d04479eb54032b790cb51a14b778e" member_type: VOTER last_known_addr { host: "127.6.242.124" port: 34319 } } }
04:02:50.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.784616 7395 sys_catalog.cc:458] T 00000000000000000000000000000000 P f21e4c0341ef4a30bb9d8f056b478ec8 [sys.catalog]: This master's current role is: FOLLOWER
04:02:50.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.822177 7398 mvcc.cc:205] Tried to move back new op lower bound from 7334167225537392640 to 7334167225344454656. Current Snapshot: MvccSnapshot[applied={T|T < 7334167225537392640}]
04:02:50.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.843757 7396 sys_catalog.cc:455] T 00000000000000000000000000000000 P f21e4c0341ef4a30bb9d8f056b478ec8 [sys.catalog]: SysCatalogTable state changed. Reason: New leader fdd06fdfb82a4d1caf216183316d41d8. Latest consensus state: current_term: 1 leader_uuid: "fdd06fdfb82a4d1caf216183316d41d8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fdd06fdfb82a4d1caf216183316d41d8" member_type: VOTER last_known_addr { host: "127.6.242.126" port: 44287 } } peers { permanent_uuid: "f21e4c0341ef4a30bb9d8f056b478ec8" member_type: VOTER last_known_addr { host: "127.6.242.125" port: 39089 } } peers { permanent_uuid: "cb0d04479eb54032b790cb51a14b778e" member_type: VOTER last_known_addr { host: "127.6.242.124" port: 34319 } } }
04:02:50.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.844079 7395 sys_catalog.cc:455] T 00000000000000000000000000000000 P f21e4c0341ef4a30bb9d8f056b478ec8 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "fdd06fdfb82a4d1caf216183316d41d8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fdd06fdfb82a4d1caf216183316d41d8" member_type: VOTER last_known_addr { host: "127.6.242.126" port: 44287 } } peers { permanent_uuid: "f21e4c0341ef4a30bb9d8f056b478ec8" member_type: VOTER last_known_addr { host: "127.6.242.125" port: 39089 } } peers { permanent_uuid: "cb0d04479eb54032b790cb51a14b778e" member_type: VOTER last_known_addr { host: "127.6.242.124" port: 34319 } } }
04:02:50.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.845639 7396 sys_catalog.cc:458] T 00000000000000000000000000000000 P f21e4c0341ef4a30bb9d8f056b478ec8 [sys.catalog]: This master's current role is: FOLLOWER
04:02:50.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.845669 7395 sys_catalog.cc:458] T 00000000000000000000000000000000 P f21e4c0341ef4a30bb9d8f056b478ec8 [sys.catalog]: This master's current role is: FOLLOWER
04:02:50.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:50.898175 7414 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
04:02:50.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:50.938937 7392 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
04:02:50.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:50.939594 7392 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
04:02:50.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:50.940237 7392 flags.cc:425] Enabled unsafe flag: --never_fsync=true
04:02:50.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:50.991567 7392 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
04:02:50.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:50.992970 7392 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.6.242.65
04:02:51.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:51.053851 7392 tablet_server_runner.cc:78] Tablet server non-default flags:
04:02:51.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:02:51.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.242.84:45421
04:02:51.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:02:51.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:02:51.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/ts-0/data
04:02:51.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/ts-0/wal
04:02:51.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:02:51.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:02:51.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.242.65:0
04:02:51.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
04:02:51.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:02:51.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:02:51.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/ts-0/data/info.pb
04:02:51.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.242.65
04:02:51.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:02:51.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.242.126:44287,127.6.242.125:39089,127.6.242.124:34319
04:02:51.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
04:02:51.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:02:51.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
04:02:51.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
04:02:51.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:02:51.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.242.65
04:02:51.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/ts-0/logs
04:02:51.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:02:51.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
04:02:51.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
04:02:51.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
04:02:51.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
04:02:51.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
04:02:51.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
04:02:51.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 02:50:13 UTC on ccycloud.quasar-purgqw.root.comops.site
04:02:51.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676527
04:02:51.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
04:02:51.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:51.055400 7392 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
04:02:51.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:51.057508 7392 file_cache.cc:493] Constructed file cache file cache with capacity 419430
04:02:51.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:51.080132 7417 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:02:51.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:51.130477 7369 catalog_manager.cc:1259] Loaded cluster ID: 2830635cd2194ae4829a4bb28c220f5d
04:02:51.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:51.131073 7369 catalog_manager.cc:1552] T 00000000000000000000000000000000 P cb0d04479eb54032b790cb51a14b778e: loading cluster ID for follower catalog manager: success
04:02:51.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:51.138947 7369 catalog_manager.cc:1574] T 00000000000000000000000000000000 P cb0d04479eb54032b790cb51a14b778e: acquiring CA information for follower catalog manager: success
04:02:51.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:51.148175 7369 catalog_manager.cc:1602] T 00000000000000000000000000000000 P cb0d04479eb54032b790cb51a14b778e: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
04:02:51.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:51.430706 7423 sys_catalog.cc:455] T 00000000000000000000000000000000 P fdd06fdfb82a4d1caf216183316d41d8 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "fdd06fdfb82a4d1caf216183316d41d8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fdd06fdfb82a4d1caf216183316d41d8" member_type: VOTER last_known_addr { host: "127.6.242.126" port: 44287 } } peers { permanent_uuid: "f21e4c0341ef4a30bb9d8f056b478ec8" member_type: VOTER last_known_addr { host: "127.6.242.125" port: 39089 } } peers { permanent_uuid: "cb0d04479eb54032b790cb51a14b778e" member_type: VOTER last_known_addr { host: "127.6.242.124" port: 34319 } } }
04:02:51.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:51.431459 7423 sys_catalog.cc:458] T 00000000000000000000000000000000 P fdd06fdfb82a4d1caf216183316d41d8 [sys.catalog]: This master's current role is: LEADER
04:02:51.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:51.934285 7414 catalog_manager.cc:1259] Loaded cluster ID: 2830635cd2194ae4829a4bb28c220f5d
04:02:51.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:51.934654 7414 catalog_manager.cc:1552] T 00000000000000000000000000000000 P f21e4c0341ef4a30bb9d8f056b478ec8: loading cluster ID for follower catalog manager: success
04:02:51.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:51.952034 7414 catalog_manager.cc:1574] T 00000000000000000000000000000000 P f21e4c0341ef4a30bb9d8f056b478ec8: acquiring CA information for follower catalog manager: success
04:02:51.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:51.989917 7414 catalog_manager.cc:1602] T 00000000000000000000000000000000 P f21e4c0341ef4a30bb9d8f056b478ec8: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
04:02:53.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:52.474769 7416 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 7392
04:02:53.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:53.722661 7416 kernel_stack_watchdog.cc:198] Thread 7392 stuck at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:637 for 393ms:
04:02:53.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
04:02:53.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
04:02:53.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
04:02:53.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
04:02:53.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:51.081341 7418 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:02:53.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:53.724016 7419 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
04:02:53.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:53.724171 7392 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 2.645s user 0.640s sys 1.197s
04:02:53.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:53.725497 7420 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:02:53.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:53.725956 7392 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 2.646s user 0.640s sys 1.198s
04:02:53.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:53.726258 7392 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
04:02:53.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:53.729823 7392 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
04:02:53.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:53.742040 7392 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
04:02:53.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:53.745867 7392 hybrid_clock.cc:648] HybridClock initialized: now 1790568173745766 us; error 104 us; skew 500 ppm
04:02:53.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:53.747115 7392 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
04:02:53.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:53.777371 7392 webserver.cc:460] Webserver started at http://127.6.242.65:46033/ using document root <none> and password file <none>
04:02:53.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:53.778949 7392 fs_manager.cc:359] Metadata directory not provided
04:02:53.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:53.779498 7392 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
04:02:53.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:53.780216 7392 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
04:02:53.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:53.787580 7392 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/ts-0/data/instance:
04:02:53.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "595a14b03bab40b687f11b69de9e79d1"
04:02:53.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 04:02:53 on dist-test-slave-jlk3"
04:02:53.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:53.789690 7392 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/ts-0/wal/instance:
04:02:53.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "595a14b03bab40b687f11b69de9e79d1"
04:02:53.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 04:02:53 on dist-test-slave-jlk3"
04:02:53.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:53.814823 7392 fs_manager.cc:691] Time spent creating directory manager: real 0.024s user 0.015s sys 0.007s
04:02:53.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:53.833671 7434 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
04:02:53.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:53.837723 7392 fs_manager.cc:725] Time spent opening block manager: real 0.018s user 0.009s sys 0.005s
04:02:53.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:53.838389 7392 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/ts-0/data,/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/ts-0/wal
04:02:53.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "595a14b03bab40b687f11b69de9e79d1"
04:02:53.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 04:02:53 on dist-test-slave-jlk3"
04:02:53.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:53.839097 7392 fs_report.cc:383] FS layout report
04:02:53.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
04:02:53.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/ts-0/wal
04:02:53.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/ts-0/wal
04:02:53.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/ts-0/data/data
04:02:53.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
04:02:53.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
04:02:53.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
04:02:53.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
04:02:53.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
04:02:53.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
04:02:53.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
04:02:53.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
04:02:53.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
04:02:53.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
04:02:53.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
04:02:53.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:53.919194 7392 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
04:02:53.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:53.921602 7392 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
04:02:53.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:53.922498 7392 kserver.cc:163] Server-wide thread pool size limit: 3276
04:02:53.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:53.932464 7392 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
04:02:53.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:53.945068 7392 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
04:02:53.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:53.945573 7392 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.001s user 0.001s sys 0.000s
04:02:53.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:53.946049 7392 ts_tablet_manager.cc:610] Registered 0 tablets
04:02:53.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:53.946388 7392 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
04:02:53.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:53.992363 7343 debug-util.cc:398] Leaking SignalData structure 0x7b08000b8520 after lost signal to thread 7275
04:02:53.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:53.993324 7343 debug-util.cc:398] Leaking SignalData structure 0x7b08000ba240 after lost signal to thread 7346
04:02:54.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:54.823361 7548 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.242.65:40385 every 8 connection(s)
04:02:54.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:54.823902 7392 rpc_server.cc:307] RPC server started. Bound to: 127.6.242.65:40385
04:02:54.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:54.827437 7392 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/ts-0/data/info.pb
04:02:54.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:54.835466 7113 external_mini_cluster.cc:1388] Started /tmp/dist-test-task4Qe23I/build/tsan/bin/kudu as pid 7392
04:02:54.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:54.836092 7113 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/ts-0/wal/instance
04:02:54.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:54.853860 7113 external_mini_cluster.cc:1326] Running /tmp/dist-test-task4Qe23I/build/tsan/bin/kudu
04:02:54.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4Qe23I/build/tsan/bin/kudu
04:02:54.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/ts-1/wal
04:02:54.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/ts-1/data
04:02:54.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
04:02:54.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
04:02:54.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
04:02:54.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:02:54.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:02:54.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:02:54.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/ts-1/logs
04:02:54.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/ts-1/data/info.pb
04:02:54.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:02:54.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
04:02:54.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
04:02:54.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
04:02:54.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
04:02:54.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:02:54.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:02:54.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:02:54.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
04:02:54.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
04:02:54.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.242.66:0
04:02:54.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.242.66
04:02:54.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.242.66
04:02:54.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:02:54.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.242.126:44287,127.6.242.125:39089,127.6.242.124:34319
04:02:54.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.242.84:45421
04:02:54.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:02:54.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:02:54.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
04:02:54.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:54.920504 7549 heartbeater.cc:347] Connected to a master server at 127.6.242.124:34319
04:02:54.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:54.921002 7549 heartbeater.cc:464] Registering TS with master...
04:02:54.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:54.922183 7549 heartbeater.cc:511] Master 127.6.242.124:34319 requested a full tablet report, sending...
04:02:54.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:54.926734 7312 ts_manager.cc:194] Registered new tserver with Master: 595a14b03bab40b687f11b69de9e79d1 (127.6.242.65:40385)
04:02:54.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:54.941673 7556 heartbeater.cc:347] Connected to a master server at 127.6.242.126:44287
04:02:54.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:54.942020 7556 heartbeater.cc:464] Registering TS with master...
04:02:54.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:54.942804 7556 heartbeater.cc:511] Master 127.6.242.126:44287 requested a full tablet report, sending...
04:02:54.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:54.944402 7550 heartbeater.cc:347] Connected to a master server at 127.6.242.125:39089
04:02:54.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:54.944736 7550 heartbeater.cc:464] Registering TS with master...
04:02:54.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:54.945385 7158 ts_manager.cc:194] Registered new tserver with Master: 595a14b03bab40b687f11b69de9e79d1 (127.6.242.65:40385)
04:02:54.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:54.945518 7550 heartbeater.cc:511] Master 127.6.242.125:39089 requested a full tablet report, sending...
04:02:54.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:54.949827 7233 ts_manager.cc:194] Registered new tserver with Master: 595a14b03bab40b687f11b69de9e79d1 (127.6.242.65:40385)
04:02:54.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:54.951352 7158 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.6.242.65:48589
04:02:55.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:55.316799 7553 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
04:02:55.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:55.317399 7553 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
04:02:55.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:55.318012 7553 flags.cc:425] Enabled unsafe flag: --never_fsync=true
04:02:55.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:55.349305 7553 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
04:02:55.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:55.350243 7553 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.6.242.66
04:02:55.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:55.385191 7553 tablet_server_runner.cc:78] Tablet server non-default flags:
04:02:55.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:02:55.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.242.84:45421
04:02:55.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:02:55.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:02:55.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/ts-1/data
04:02:55.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/ts-1/wal
04:02:55.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:02:55.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:02:55.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.242.66:0
04:02:55.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
04:02:55.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:02:55.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:02:55.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/ts-1/data/info.pb
04:02:55.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.242.66
04:02:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:02:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.242.126:44287,127.6.242.125:39089,127.6.242.124:34319
04:02:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
04:02:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:02:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
04:02:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
04:02:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:02:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.242.66
04:02:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/ts-1/logs
04:02:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:02:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
04:02:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
04:02:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
04:02:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
04:02:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
04:02:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
04:02:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 02:50:13 UTC on ccycloud.quasar-purgqw.root.comops.site
04:02:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676527
04:02:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
04:02:55.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:55.386744 7553 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
04:02:55.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:55.388818 7553 file_cache.cc:493] Constructed file cache file cache with capacity 419430
04:02:55.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:55.418612 7567 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:02:55.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:55.955605 7556 heartbeater.cc:503] Master 127.6.242.126:44287 was elected leader, sending a full tablet report...
04:02:57.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:56.811034 7565 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 7553
04:02:57.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:57.334935 7565 kernel_stack_watchdog.cc:198] Thread 7553 stuck at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:637 for 391ms:
04:02:57.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
04:02:57.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
04:02:57.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
04:02:57.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
04:02:57.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:55.419655 7566 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:02:57.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:57.336566 7553 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 1.920s user 0.557s sys 0.963s
04:02:57.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:57.336907 7553 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 1.920s user 0.558s sys 0.963s
04:02:57.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:57.340175 7568 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1918 milliseconds
04:02:57.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:57.341703 7569 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:02:57.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:57.341749 7553 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
04:02:57.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:57.342991 7553 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
04:02:57.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:57.345242 7553 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
04:02:57.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:57.346642 7553 hybrid_clock.cc:648] HybridClock initialized: now 1790568177346570 us; error 68 us; skew 500 ppm
04:02:57.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:57.347600 7553 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
04:02:57.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:57.353699 7553 webserver.cc:460] Webserver started at http://127.6.242.66:43841/ using document root <none> and password file <none>
04:02:57.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:57.354619 7553 fs_manager.cc:359] Metadata directory not provided
04:02:57.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:57.354864 7553 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
04:02:57.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:57.355350 7553 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
04:02:57.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:57.360224 7553 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/ts-1/data/instance:
04:02:57.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "52d37e53dc384ab290b21fa0a9e182eb"
04:02:57.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 04:02:57 on dist-test-slave-jlk3"
04:02:57.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:57.361606 7553 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/ts-1/wal/instance:
04:02:57.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "52d37e53dc384ab290b21fa0a9e182eb"
04:02:57.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 04:02:57 on dist-test-slave-jlk3"
04:02:57.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:57.368932 7553 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.002s
04:02:57.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:57.374490 7577 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
04:02:57.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:57.375552 7553 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
04:02:57.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:57.375910 7553 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/ts-1/data,/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/ts-1/wal
04:02:57.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "52d37e53dc384ab290b21fa0a9e182eb"
04:02:57.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 04:02:57 on dist-test-slave-jlk3"
04:02:57.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:57.376264 7553 fs_report.cc:383] FS layout report
04:02:57.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
04:02:57.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/ts-1/wal
04:02:57.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/ts-1/wal
04:02:57.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/ts-1/data/data
04:02:57.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
04:02:57.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
04:02:57.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
04:02:57.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
04:02:57.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
04:02:57.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
04:02:57.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
04:02:57.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
04:02:57.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
04:02:57.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
04:02:57.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
04:02:57.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:57.420009 7553 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
04:02:57.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:57.421784 7553 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
04:02:57.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:57.422329 7553 kserver.cc:163] Server-wide thread pool size limit: 3276
04:02:57.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:57.425024 7553 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
04:02:57.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:57.429774 7553 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
04:02:57.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:57.430007 7553 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
04:02:57.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:57.430349 7553 ts_tablet_manager.cc:610] Registered 0 tablets
04:02:57.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:57.430534 7553 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
04:02:57.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:57.574864 7553 rpc_server.cc:307] RPC server started. Bound to: 127.6.242.66:34813
04:02:57.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:57.574955 7689 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.242.66:34813 every 8 connection(s)
04:02:57.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:57.577485 7553 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/ts-1/data/info.pb
04:02:57.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:57.588356 7113 external_mini_cluster.cc:1388] Started /tmp/dist-test-task4Qe23I/build/tsan/bin/kudu as pid 7553
04:02:57.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:57.588836 7113 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/ts-1/wal/instance
04:02:57.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:57.598758 7113 external_mini_cluster.cc:1326] Running /tmp/dist-test-task4Qe23I/build/tsan/bin/kudu
04:02:57.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4Qe23I/build/tsan/bin/kudu
04:02:57.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/ts-2/wal
04:02:57.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/ts-2/data
04:02:57.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
04:02:57.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
04:02:57.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
04:02:57.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:02:57.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:02:57.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:02:57.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/ts-2/logs
04:02:57.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/ts-2/data/info.pb
04:02:57.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:02:57.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
04:02:57.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
04:02:57.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
04:02:57.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
04:02:57.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:02:57.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:02:57.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:02:57.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
04:02:57.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
04:02:57.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.242.67:0
04:02:57.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.242.67
04:02:57.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.242.67
04:02:57.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:02:57.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.242.126:44287,127.6.242.125:39089,127.6.242.124:34319
04:02:57.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.242.84:45421
04:02:57.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:02:57.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:02:57.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
04:02:57.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:57.619971 7690 heartbeater.cc:347] Connected to a master server at 127.6.242.124:34319
04:02:57.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:57.620242 7691 heartbeater.cc:347] Connected to a master server at 127.6.242.125:39089
04:02:57.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:57.620474 7690 heartbeater.cc:464] Registering TS with master...
04:02:57.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:57.620566 7691 heartbeater.cc:464] Registering TS with master...
04:02:57.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:57.621549 7691 heartbeater.cc:511] Master 127.6.242.125:39089 requested a full tablet report, sending...
04:02:57.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:57.622941 7690 heartbeater.cc:511] Master 127.6.242.124:34319 requested a full tablet report, sending...
04:02:57.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:57.624526 7692 heartbeater.cc:347] Connected to a master server at 127.6.242.126:44287
04:02:57.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:57.624836 7692 heartbeater.cc:464] Registering TS with master...
04:02:57.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:57.625560 7233 ts_manager.cc:194] Registered new tserver with Master: 52d37e53dc384ab290b21fa0a9e182eb (127.6.242.66:34813)
04:02:57.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:57.625653 7692 heartbeater.cc:511] Master 127.6.242.126:44287 requested a full tablet report, sending...
04:02:57.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:57.625859 7312 ts_manager.cc:194] Registered new tserver with Master: 52d37e53dc384ab290b21fa0a9e182eb (127.6.242.66:34813)
04:02:57.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:57.627903 7158 ts_manager.cc:194] Registered new tserver with Master: 52d37e53dc384ab290b21fa0a9e182eb (127.6.242.66:34813)
04:02:57.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:57.629984 7158 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.6.242.66:35027
04:02:57.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:57.907289 7700 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
04:02:57.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:57.907750 7700 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
04:02:57.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:57.908170 7700 flags.cc:425] Enabled unsafe flag: --never_fsync=true
04:02:57.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:57.938947 7700 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
04:02:57.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:57.939740 7700 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.6.242.67
04:02:57.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:57.978340 7700 tablet_server_runner.cc:78] Tablet server non-default flags:
04:02:57.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:02:57.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.242.84:45421
04:02:57.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:02:57.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:02:57.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/ts-2/data
04:02:57.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/ts-2/wal
04:02:57.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:02:57.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:02:57.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.242.67:0
04:02:57.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
04:02:57.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:02:57.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:02:57.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/ts-2/data/info.pb
04:02:57.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.242.67
04:02:57.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:02:57.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.242.126:44287,127.6.242.125:39089,127.6.242.124:34319
04:02:57.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
04:02:57.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:02:57.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
04:02:57.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
04:02:57.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:02:57.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.242.67
04:02:57.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/ts-2/logs
04:02:57.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:02:57.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
04:02:57.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
04:02:57.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
04:02:57.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
04:02:57.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
04:02:57.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
04:02:57.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 02:50:13 UTC on ccycloud.quasar-purgqw.root.comops.site
04:02:57.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676527
04:02:57.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
04:02:57.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:57.979451 7700 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
04:02:57.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:57.980921 7700 file_cache.cc:493] Constructed file cache file cache with capacity 419430
04:02:58.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:58.000329 7707 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:02:58.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:58.633496 7692 heartbeater.cc:503] Master 127.6.242.126:44287 was elected leader, sending a full tablet report...
04:02:59.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:58.000326 7709 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:02:59.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:58.000398 7706 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:02:59.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:02:59.225780 7708 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
04:02:59.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:59.226095 7700 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
04:02:59.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:59.229871 7700 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
04:02:59.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:59.232743 7700 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
04:02:59.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:59.234205 7700 hybrid_clock.cc:648] HybridClock initialized: now 1790568179234142 us; error 80 us; skew 500 ppm
04:02:59.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:59.234958 7700 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
04:02:59.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:59.241955 7700 webserver.cc:460] Webserver started at http://127.6.242.67:42137/ using document root <none> and password file <none>
04:02:59.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:59.242803 7700 fs_manager.cc:359] Metadata directory not provided
04:02:59.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:59.242980 7700 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
04:02:59.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:59.243361 7700 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
04:02:59.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:59.248013 7700 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/ts-2/data/instance:
04:02:59.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ac49b4c8cbda4b20b76e81d5a80dd6fb"
04:02:59.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 04:02:59 on dist-test-slave-jlk3"
04:02:59.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:59.249286 7700 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/ts-2/wal/instance:
04:02:59.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ac49b4c8cbda4b20b76e81d5a80dd6fb"
04:02:59.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 04:02:59 on dist-test-slave-jlk3"
04:02:59.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:59.256256 7700 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.006s sys 0.000s
04:02:59.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:59.261790 7717 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
04:02:59.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:59.263008 7700 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
04:02:59.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:59.263285 7700 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/ts-2/data,/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/ts-2/wal
04:02:59.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ac49b4c8cbda4b20b76e81d5a80dd6fb"
04:02:59.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 04:02:59 on dist-test-slave-jlk3"
04:02:59.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:59.263578 7700 fs_report.cc:383] FS layout report
04:02:59.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
04:02:59.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/ts-2/wal
04:02:59.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/ts-2/wal
04:02:59.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/ts-2/data/data
04:02:59.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
04:02:59.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
04:02:59.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
04:02:59.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
04:02:59.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
04:02:59.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
04:02:59.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
04:02:59.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
04:02:59.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
04:02:59.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
04:02:59.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
04:02:59.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:59.303740 7700 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
04:02:59.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:59.305207 7700 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
04:02:59.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:59.305594 7700 kserver.cc:163] Server-wide thread pool size limit: 3276
04:02:59.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:59.307928 7700 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
04:02:59.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:59.312068 7700 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
04:02:59.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:59.312263 7700 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
04:02:59.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:59.312469 7700 ts_tablet_manager.cc:610] Registered 0 tablets
04:02:59.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:59.312587 7700 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
04:02:59.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:59.444593 7700 rpc_server.cc:307] RPC server started. Bound to: 127.6.242.67:41083
04:02:59.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:59.444720 7829 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.242.67:41083 every 8 connection(s)
04:02:59.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:59.446978 7700 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/ts-2/data/info.pb
04:02:59.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:59.456132 7113 external_mini_cluster.cc:1388] Started /tmp/dist-test-task4Qe23I/build/tsan/bin/kudu as pid 7700
04:02:59.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:59.456583 7113 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster2317600421157823470/ts-2/wal/instance
04:02:59.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:59.489338 7831 heartbeater.cc:347] Connected to a master server at 127.6.242.125:39089
04:02:59.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:59.489799 7831 heartbeater.cc:464] Registering TS with master...
04:02:59.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:59.490391 7830 heartbeater.cc:347] Connected to a master server at 127.6.242.124:34319
04:02:59.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:59.490675 7830 heartbeater.cc:464] Registering TS with master...
04:02:59.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:59.490943 7831 heartbeater.cc:511] Master 127.6.242.125:39089 requested a full tablet report, sending...
04:02:59.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:59.491498 7830 heartbeater.cc:511] Master 127.6.242.124:34319 requested a full tablet report, sending...
04:02:59.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:59.494400 7232 ts_manager.cc:194] Registered new tserver with Master: ac49b4c8cbda4b20b76e81d5a80dd6fb (127.6.242.67:41083)
04:02:59.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:59.494673 7312 ts_manager.cc:194] Registered new tserver with Master: ac49b4c8cbda4b20b76e81d5a80dd6fb (127.6.242.67:41083)
04:02:59.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:59.498152 7832 heartbeater.cc:347] Connected to a master server at 127.6.242.126:44287
04:02:59.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:59.498462 7832 heartbeater.cc:464] Registering TS with master...
04:02:59.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:59.499086 7832 heartbeater.cc:511] Master 127.6.242.126:44287 requested a full tablet report, sending...
04:02:59.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:59.500857 7158 ts_manager.cc:194] Registered new tserver with Master: ac49b4c8cbda4b20b76e81d5a80dd6fb (127.6.242.67:41083)
04:02:59.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:59.501941 7113 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
04:02:59.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:02:59.502508 7158 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.6.242.67:48655
04:02:59.513 [DEBUG - main] (MiniKuduCluster.java:195) Response:
04:02:59.514 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
04:02:59.524 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.6.242.126"
port: 44287
}
bound_http_address {
host: "127.6.242.126"
port: 35835
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.6.242.125"
port: 39089
}
bound_http_address {
host: "127.6.242.125"
port: 35807
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.6.242.124"
port: 34319
}
bound_http_address {
host: "127.6.242.124"
port: 41567
}
}
}
04:02:59.530 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
04:02:59.532 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.6.242.65"
port: 40385
}
bound_http_address {
host: "127.6.242.65"
port: 46033
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.6.242.66"
port: 34813
}
bound_http_address {
host: "127.6.242.66"
port: 43841
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.6.242.67"
port: 41083
}
bound_http_address {
host: "127.6.242.67"
port: 42137
}
}
}
04:02:59.533 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
04:02:59.759 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
04:02:59.860 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task4Qe23I/test-tmp
04:02:59.862 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
04:02:59.862 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task4Qe23I/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
04:02:59.869 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
04:03:00.193 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
04:03:00.194 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
04:03:00.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:00.197381 7841 mini_oidc.cc:152] Starting JWKS server
04:03:00.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:00.200635 7841 webserver.cc:460] Webserver started at http://127.0.0.1:34185/ using document root <none> and password file <none>
04:03:00.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:00.201092 7841 mini_oidc.cc:189] Starting OIDC Discovery server
04:03:00.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:00.202687 7841 webserver.cc:460] Webserver started at http://127.0.0.1:36705/ using document root <none> and password file <none>
04:03:00.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
04:03:00.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/krb5kdc/principal' for realm 'KRBTEST.COM',
04:03:00.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
04:03:00.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 04:03:00 dist-test-slave-jlk3 krb5kdc[7853](info): setting up network...
04:03:00.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
04:03:00.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 04:03:00 dist-test-slave-jlk3 krb5kdc[7853](info): set up 2 sockets
04:03:00.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 04:03:00 dist-test-slave-jlk3 krb5kdc[7853](info): commencing operation
04:03:00.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
04:03:00.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:00.506557 7832 heartbeater.cc:503] Master 127.6.242.126:44287 was elected leader, sending a full tablet report...
04:03:01.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:01.315462 7841 mini_kdc.cc:121] Time spent starting KDC: real 1.082s user 0.000s sys 0.019s
04:03:01.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
04:03:01.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
04:03:01.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
04:03:01.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
04:03:01.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
04:03:01.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
04:03:01.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
04:03:01.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
04:03:01.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
04:03:01.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
04:03:01.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/krb5kdc/test-user.keytab.
04:03:01.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/krb5kdc/test-user.keytab.
04:03:01.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 04:03:01 dist-test-slave-jlk3 krb5kdc[7853](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790568181, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
04:03:01.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
04:03:01.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-28T04:03:01Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
04:03:01.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-28T04:03:01Z Disabled control of system clock
04:03:01.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.7.168.126@KRBTEST.COM; defaulting to no policy
04:03:01.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
04:03:01.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.7.168.126@KRBTEST.COM" created.
04:03:01.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
04:03:01.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.7.168.126 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/krb5kdc/kudu_127.7.168.126.keytab.
04:03:01.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.7.168.126 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/krb5kdc/kudu_127.7.168.126.keytab.
04:03:01.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:01.508881 7841 external_mini_cluster.cc:1326] Running /tmp/dist-test-task4Qe23I/build/tsan/bin/kudu
04:03:01.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4Qe23I/build/tsan/bin/kudu
04:03:01.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-0/wal
04:03:01.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-0/data
04:03:01.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
04:03:01.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
04:03:01.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
04:03:01.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:03:01.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:03:01.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:03:01.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-0/logs
04:03:01.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-0/data/info.pb
04:03:01.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:03:01.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
04:03:01.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
04:03:01.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
04:03:01.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
04:03:01.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:03:01.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:03:01.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:03:01.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
04:03:01.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
04:03:01.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:03:01.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:03:01.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.168.126:46569
04:03:01.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.168.126
04:03:01.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:03:01.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/krb5kdc/kudu_127.7.168.126.keytab
04:03:01.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.168.126
04:03:01.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
04:03:01.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
04:03:01.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
04:03:01.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.168.84:39921
04:03:01.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:03:01.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:03:01.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:03:01.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:03:01.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393
04:03:01.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:03:01.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34185/jwks.json
04:03:01.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
04:03:01.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:01.807185 7868 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
04:03:01.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:01.807925 7868 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
04:03:01.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:01.808665 7868 flags.cc:425] Enabled unsafe flag: --never_fsync=true
04:03:01.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:01.839671 7868 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
04:03:01.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:01.839920 7868 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
04:03:01.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:01.840225 7868 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
04:03:01.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:01.840430 7868 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
04:03:01.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:01.840613 7868 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
04:03:01.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:01.840863 7868 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
04:03:01.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:01.841008 7868 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34185/jwks.json
04:03:01.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:01.876165 7868 master_runner.cc:386] Master server non-default flags:
04:03:01.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:03:01.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.168.84:39921
04:03:01.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:03:01.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:03:01.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-0/data
04:03:01.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-0/wal
04:03:01.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
04:03:01.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:03:01.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393
04:03:01.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/krb5kdc/kudu_127.7.168.126.keytab
04:03:01.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.168.126
04:03:01.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:03:01.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:03:01.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:03:01.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.168.126:46569
04:03:01.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:03:01.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
04:03:01.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:03:01.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34185/jwks.json
04:03:01.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
04:03:01.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
04:03:01.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
04:03:01.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:03:01.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:03:01.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-0/data/info.pb
04:03:01.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.168.126
04:03:01.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:03:01.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
04:03:01.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:03:01.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
04:03:01.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
04:03:01.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:03:01.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-0/logs
04:03:01.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:03:01.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
04:03:01.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
04:03:01.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
04:03:01.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
04:03:01.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
04:03:01.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
04:03:01.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 02:50:13 UTC on ccycloud.quasar-purgqw.root.comops.site
04:03:01.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676527
04:03:01.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
04:03:01.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:01.877410 7868 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
04:03:01.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:01.878330 7868 file_cache.cc:493] Constructed file cache file cache with capacity 419430
04:03:01.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:01.896858 7875 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:03:03.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:01.896934 7877 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:03:03.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:01.897212 7874 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:03:03.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:01.900765 7868 server_base.cc:1034] running on GCE node
04:03:03.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:03.118227 7868 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
04:03:03.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:03.120610 7868 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
04:03:03.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:03.122066 7868 hybrid_clock.cc:648] HybridClock initialized: now 1790568183122045 us; error 41 us; skew 500 ppm
04:03:03.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:03.122813 7868 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
04:03:03.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 04:03:03 dist-test-slave-jlk3 krb5kdc[7853](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790568183, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.168.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
04:03:03.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:03.128118 7868 init.cc:377] Logged in from keytab as kudu/127.7.168.126@KRBTEST.COM (short username kudu)
04:03:03.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:03.132858 7868 webserver.cc:460] Webserver started at http://127.7.168.126:34837/ using document root <none> and password file <none>
04:03:03.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:03.133679 7868 fs_manager.cc:359] Metadata directory not provided
04:03:03.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:03.133860 7868 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
04:03:03.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:03.134217 7868 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
04:03:03.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:03.138962 7868 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-0/data/instance:
04:03:03.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f67c27ee2e584d25850a0572b8a3b27c"
04:03:03.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 04:03:03 on dist-test-slave-jlk3"
04:03:03.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:03.140015 7868 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-0/wal/instance:
04:03:03.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f67c27ee2e584d25850a0572b8a3b27c"
04:03:03.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 04:03:03 on dist-test-slave-jlk3"
04:03:03.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:03.147055 7868 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.001s sys 0.008s
04:03:03.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:03.152627 7886 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
04:03:03.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:03.153657 7868 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
04:03:03.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:03.154017 7868 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-0/data,/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-0/wal
04:03:03.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f67c27ee2e584d25850a0572b8a3b27c"
04:03:03.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 04:03:03 on dist-test-slave-jlk3"
04:03:03.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:03.154356 7868 fs_report.cc:383] FS layout report
04:03:03.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
04:03:03.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-0/wal
04:03:03.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-0/wal
04:03:03.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-0/data/data
04:03:03.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
04:03:03.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
04:03:03.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
04:03:03.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
04:03:03.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
04:03:03.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
04:03:03.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
04:03:03.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
04:03:03.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
04:03:03.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
04:03:03.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
04:03:03.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:03.203824 7868 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
04:03:03.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:03.210609 7868 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
04:03:03.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:03.211126 7868 kserver.cc:163] Server-wide thread pool size limit: 3276
04:03:03.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:03.291263 7868 rpc_server.cc:307] RPC server started. Bound to: 127.7.168.126:46569
04:03:03.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:03.291343 7937 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.168.126:46569 every 8 connection(s)
04:03:03.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:03.293843 7868 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-0/data/info.pb
04:03:03.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:03.298797 7841 external_mini_cluster.cc:1388] Started /tmp/dist-test-task4Qe23I/build/tsan/bin/kudu as pid 7868
04:03:03.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:03.299321 7841 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-0/wal/instance
04:03:03.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:03.300398 7938 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
04:03:03.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:03.314638 7938 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.168.126" port: 46569 } has no permanent_uuid. Determining permanent_uuid...
04:03:03.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.7.168.125@KRBTEST.COM; defaulting to no policy
04:03:03.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
04:03:03.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.7.168.125@KRBTEST.COM" created.
04:03:03.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 04:03:03 dist-test-slave-jlk3 krb5kdc[7853](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790568183, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.168.126@KRBTEST.COM for kudu/127.7.168.126@KRBTEST.COM
04:03:03.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
04:03:03.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.7.168.125 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/krb5kdc/kudu_127.7.168.125.keytab.
04:03:03.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.7.168.125 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/krb5kdc/kudu_127.7.168.125.keytab.
04:03:03.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:03.344921 7841 external_mini_cluster.cc:1326] Running /tmp/dist-test-task4Qe23I/build/tsan/bin/kudu
04:03:03.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4Qe23I/build/tsan/bin/kudu
04:03:03.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-1/wal
04:03:03.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-1/data
04:03:03.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
04:03:03.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
04:03:03.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
04:03:03.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:03:03.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:03:03.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:03:03.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-1/logs
04:03:03.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-1/data/info.pb
04:03:03.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:03:03.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
04:03:03.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
04:03:03.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
04:03:03.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
04:03:03.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:03:03.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:03:03.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:03:03.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
04:03:03.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
04:03:03.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:03:03.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:03:03.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.168.125:46485
04:03:03.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.168.125
04:03:03.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:03:03.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/krb5kdc/kudu_127.7.168.125.keytab
04:03:03.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.168.125
04:03:03.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
04:03:03.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
04:03:03.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
04:03:03.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.168.84:39921
04:03:03.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:03:03.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:03:03.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:03:03.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:03:03.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393
04:03:03.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:03:03.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34185/jwks.json
04:03:03.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
04:03:03.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:03.354660 7938 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.168.125" port: 46485 } has no permanent_uuid. Determining permanent_uuid...
04:03:03.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:03.356652 7888 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.168.125:46485: connect: Connection refused (error 111)
04:03:03.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:03.361034 7938 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.168.125:46485: Network error: Client connection negotiation failed: client connection to 127.7.168.125:46485: connect: Connection refused (error 111)
04:03:03.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:03.410506 7938 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.168.125" port: 46485 } attempt: 1
04:03:03.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:03.414718 7938 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.168.125:46485: Network error: Client connection negotiation failed: client connection to 127.7.168.125:46485: connect: Connection refused (error 111)
04:03:03.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:03.483065 7938 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.168.125" port: 46485 } attempt: 2
04:03:03.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:03.486970 7938 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.168.125:46485: Network error: Client connection negotiation failed: client connection to 127.7.168.125:46485: connect: Connection refused (error 111)
04:03:03.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:03.556382 7826 debug-util.cc:398] Leaking SignalData structure 0x7b08000bb0a0 after lost signal to thread 7701
04:03:03.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:03.557211 7826 debug-util.cc:398] Leaking SignalData structure 0x7b08000bb0c0 after lost signal to thread 7829
04:03:03.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:03.578293 7938 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.168.125" port: 46485 } attempt: 3
04:03:03.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:03.583323 7938 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.168.125:46485: Network error: Client connection negotiation failed: client connection to 127.7.168.125:46485: connect: Connection refused (error 111)
04:03:03.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:03.666251 7944 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
04:03:03.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:03.666878 7944 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
04:03:03.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:03.667546 7944 flags.cc:425] Enabled unsafe flag: --never_fsync=true
04:03:03.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:03.699925 7944 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
04:03:03.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:03.700193 7944 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
04:03:03.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:03.700623 7944 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
04:03:03.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:03.700989 7944 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
04:03:03.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:03.701301 7944 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
04:03:03.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:03.701606 7944 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
04:03:03.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:03.701939 7944 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34185/jwks.json
04:03:03.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:03.726715 7938 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.168.125" port: 46485 } attempt: 4
04:03:03.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:03.730779 7938 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.168.125:46485: Network error: Client connection negotiation failed: client connection to 127.7.168.125:46485: connect: Connection refused (error 111)
04:03:03.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:03.739078 7944 master_runner.cc:386] Master server non-default flags:
04:03:03.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:03:03.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.168.84:39921
04:03:03.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:03:03.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:03:03.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-1/data
04:03:03.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-1/wal
04:03:03.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
04:03:03.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:03:03.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393
04:03:03.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/krb5kdc/kudu_127.7.168.125.keytab
04:03:03.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.168.125
04:03:03.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:03:03.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:03:03.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:03:03.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.168.125:46485
04:03:03.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:03:03.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
04:03:03.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:03:03.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34185/jwks.json
04:03:03.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
04:03:03.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
04:03:03.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
04:03:03.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:03:03.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:03:03.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-1/data/info.pb
04:03:03.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.168.125
04:03:03.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:03:03.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
04:03:03.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:03:03.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
04:03:03.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
04:03:03.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:03:03.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-1/logs
04:03:03.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:03:03.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
04:03:03.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
04:03:03.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
04:03:03.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
04:03:03.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
04:03:03.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
04:03:03.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 02:50:13 UTC on ccycloud.quasar-purgqw.root.comops.site
04:03:03.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676527
04:03:03.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
04:03:03.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:03.740278 7944 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
04:03:03.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:03.741670 7944 file_cache.cc:493] Constructed file cache file cache with capacity 419430
04:03:03.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:03.763906 7952 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:03:04.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:04.030185 7938 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.168.125" port: 46485 } attempt: 5
04:03:04.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:04.036756 7938 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.168.125:46485: Network error: Client connection negotiation failed: client connection to 127.7.168.125:46485: connect: Connection refused (error 111)
04:03:04.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:04.584299 7938 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.168.125" port: 46485 } attempt: 6
04:03:04.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:04.627125 7938 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.168.125:46485: Network error: Client connection negotiation failed: client connection to 127.7.168.125:46485: connect: Connection refused (error 111)
04:03:05.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:03.765323 7954 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:03:05.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:03.764019 7951 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:03:05.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:05.311681 7953 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
04:03:05.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:05.311750 7944 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
04:03:05.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:05.313272 7944 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
04:03:05.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:05.316634 7944 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
04:03:05.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:05.318456 7944 hybrid_clock.cc:648] HybridClock initialized: now 1790568185318381 us; error 88 us; skew 500 ppm
04:03:05.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:05.319622 7944 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
04:03:05.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 04:03:05 dist-test-slave-jlk3 krb5kdc[7853](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790568185, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.168.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
04:03:05.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:05.330513 7944 init.cc:377] Logged in from keytab as kudu/127.7.168.125@KRBTEST.COM (short username kudu)
04:03:05.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:05.338680 7944 webserver.cc:460] Webserver started at http://127.7.168.125:42977/ using document root <none> and password file <none>
04:03:05.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:05.339761 7944 fs_manager.cc:359] Metadata directory not provided
04:03:05.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:05.340034 7944 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
04:03:05.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:05.340616 7944 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
04:03:05.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:05.345523 7944 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-1/data/instance:
04:03:05.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "455fed1171d64bdfa974a8e29d8c171c"
04:03:05.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 04:03:05 on dist-test-slave-jlk3"
04:03:05.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:05.346807 7944 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-1/wal/instance:
04:03:05.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "455fed1171d64bdfa974a8e29d8c171c"
04:03:05.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 04:03:05 on dist-test-slave-jlk3"
04:03:05.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:05.354563 7944 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
04:03:05.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:05.360893 7967 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
04:03:05.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:05.362052 7944 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
04:03:05.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:05.362607 7944 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-1/data,/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-1/wal
04:03:05.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "455fed1171d64bdfa974a8e29d8c171c"
04:03:05.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 04:03:05 on dist-test-slave-jlk3"
04:03:05.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:05.363061 7944 fs_report.cc:383] FS layout report
04:03:05.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
04:03:05.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-1/wal
04:03:05.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-1/wal
04:03:05.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-1/data/data
04:03:05.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
04:03:05.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
04:03:05.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
04:03:05.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
04:03:05.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
04:03:05.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
04:03:05.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
04:03:05.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
04:03:05.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
04:03:05.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
04:03:05.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
04:03:05.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:05.398159 7944 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
04:03:05.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:05.404248 7944 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
04:03:05.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:05.404800 7944 kserver.cc:163] Server-wide thread pool size limit: 3276
04:03:05.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:05.498651 7944 rpc_server.cc:307] RPC server started. Bound to: 127.7.168.125:46485
04:03:05.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:05.498752 8019 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.168.125:46485 every 8 connection(s)
04:03:05.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:05.502439 7944 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-1/data/info.pb
04:03:05.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:05.505882 7841 external_mini_cluster.cc:1388] Started /tmp/dist-test-task4Qe23I/build/tsan/bin/kudu as pid 7944
04:03:05.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:05.506453 7841 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-1/wal/instance
04:03:05.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:05.509160 8020 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
04:03:05.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.7.168.124@KRBTEST.COM; defaulting to no policy
04:03:05.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
04:03:05.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.7.168.124@KRBTEST.COM" created.
04:03:05.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:05.531394 8020 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.168.126" port: 46569 } has no permanent_uuid. Determining permanent_uuid...
04:03:05.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
04:03:05.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.7.168.124 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/krb5kdc/kudu_127.7.168.124.keytab.
04:03:05.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.7.168.124 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/krb5kdc/kudu_127.7.168.124.keytab.
04:03:05.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:05.561314 7841 external_mini_cluster.cc:1326] Running /tmp/dist-test-task4Qe23I/build/tsan/bin/kudu
04:03:05.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4Qe23I/build/tsan/bin/kudu
04:03:05.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-2/wal
04:03:05.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-2/data
04:03:05.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
04:03:05.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
04:03:05.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
04:03:05.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:03:05.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:03:05.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:03:05.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-2/logs
04:03:05.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-2/data/info.pb
04:03:05.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:03:05.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
04:03:05.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
04:03:05.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
04:03:05.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
04:03:05.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:03:05.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:03:05.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:03:05.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
04:03:05.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
04:03:05.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:03:05.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:03:05.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.168.124:43393
04:03:05.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.168.124
04:03:05.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:03:05.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/krb5kdc/kudu_127.7.168.124.keytab
04:03:05.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.168.124
04:03:05.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
04:03:05.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
04:03:05.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
04:03:05.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.168.84:39921
04:03:05.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:03:05.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:03:05.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:03:05.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:03:05.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393
04:03:05.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:03:05.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34185/jwks.json
04:03:05.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
04:03:05.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 04:03:05 dist-test-slave-jlk3 krb5kdc[7853](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790568185, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.168.125@KRBTEST.COM for kudu/127.7.168.126@KRBTEST.COM
04:03:05.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:05.594743 8020 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.168.125" port: 46485 } has no permanent_uuid. Determining permanent_uuid...
04:03:05.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 04:03:05 dist-test-slave-jlk3 krb5kdc[7853](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790568185, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.168.125@KRBTEST.COM for kudu/127.7.168.125@KRBTEST.COM
04:03:05.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:05.631206 8020 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.168.124" port: 43393 } has no permanent_uuid. Determining permanent_uuid...
04:03:05.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:05.634541 7968 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.168.124:43393: connect: Connection refused (error 111)
04:03:05.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:05.638759 8020 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.168.124:43393: Network error: Client connection negotiation failed: client connection to 127.7.168.124:43393: connect: Connection refused (error 111)
04:03:05.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:05.687953 7938 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.168.125" port: 46485 } attempt: 7
04:03:05.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:05.688746 8020 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.168.124" port: 43393 } attempt: 1
04:03:05.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:05.694989 8020 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.168.124:43393: Network error: Client connection negotiation failed: client connection to 127.7.168.124:43393: connect: Connection refused (error 111)
04:03:05.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 04:03:05 dist-test-slave-jlk3 krb5kdc[7853](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790568183, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.168.126@KRBTEST.COM for kudu/127.7.168.125@KRBTEST.COM
04:03:05.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:05.724753 7938 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.168.124" port: 43393 } has no permanent_uuid. Determining permanent_uuid...
04:03:05.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:05.730738 7938 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.168.124:43393: Network error: Client connection negotiation failed: client connection to 127.7.168.124:43393: connect: Connection refused (error 111)
04:03:05.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:05.763381 8020 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.168.124" port: 43393 } attempt: 2
04:03:05.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:05.768281 8020 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.168.124:43393: Network error: Client connection negotiation failed: client connection to 127.7.168.124:43393: connect: Connection refused (error 111)
04:03:05.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:05.789156 7938 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.168.124" port: 43393 } attempt: 1
04:03:05.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:05.793678 7938 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.168.124:43393: Network error: Client connection negotiation failed: client connection to 127.7.168.124:43393: connect: Connection refused (error 111)
04:03:05.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:05.859709 8020 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.168.124" port: 43393 } attempt: 3
04:03:05.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:05.863866 8020 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.168.124:43393: Network error: Client connection negotiation failed: client connection to 127.7.168.124:43393: connect: Connection refused (error 111)
04:03:05.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:05.875084 7938 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.168.124" port: 43393 } attempt: 2
04:03:05.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:05.879844 7938 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.168.124:43393: Network error: Client connection negotiation failed: client connection to 127.7.168.124:43393: connect: Connection refused (error 111)
04:03:05.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:05.924266 8026 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
04:03:05.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:05.924944 8026 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
04:03:05.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:05.925639 8026 flags.cc:425] Enabled unsafe flag: --never_fsync=true
04:03:05.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:05.958492 8026 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
04:03:05.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:05.958763 8026 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
04:03:05.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:05.959158 8026 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
04:03:05.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:05.959473 8026 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
04:03:05.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:05.959794 8026 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
04:03:05.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:05.960117 8026 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
04:03:05.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:05.960362 8026 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34185/jwks.json
04:03:05.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:05.965315 7938 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.168.124" port: 43393 } attempt: 3
04:03:05.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:05.969406 7938 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.168.124:43393: Network error: Client connection negotiation failed: client connection to 127.7.168.124:43393: connect: Connection refused (error 111)
04:03:06.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:06.003296 8026 master_runner.cc:386] Master server non-default flags:
04:03:06.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:03:06.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.168.84:39921
04:03:06.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:03:06.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:03:06.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-2/data
04:03:06.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-2/wal
04:03:06.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
04:03:06.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:03:06.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393
04:03:06.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/krb5kdc/kudu_127.7.168.124.keytab
04:03:06.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.168.124
04:03:06.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:03:06.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:03:06.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:03:06.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.168.124:43393
04:03:06.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:03:06.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
04:03:06.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:03:06.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34185/jwks.json
04:03:06.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
04:03:06.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
04:03:06.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
04:03:06.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:03:06.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:03:06.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-2/data/info.pb
04:03:06.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.168.124
04:03:06.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:03:06.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
04:03:06.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:03:06.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
04:03:06.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
04:03:06.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:03:06.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-2/logs
04:03:06.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:03:06.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
04:03:06.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
04:03:06.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
04:03:06.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
04:03:06.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
04:03:06.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
04:03:06.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 02:50:13 UTC on ccycloud.quasar-purgqw.root.comops.site
04:03:06.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676527
04:03:06.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
04:03:06.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:06.004812 8026 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
04:03:06.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:06.006136 8026 file_cache.cc:493] Constructed file cache file cache with capacity 419430
04:03:06.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:06.007332 8020 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.168.124" port: 43393 } attempt: 4
04:03:06.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:06.016273 8020 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.168.124:43393: Network error: Client connection negotiation failed: client connection to 127.7.168.124:43393: connect: Connection refused (error 111)
04:03:06.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:06.030205 8037 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:03:06.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:06.109880 7938 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.168.124" port: 43393 } attempt: 4
04:03:06.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:06.117268 7938 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.168.124:43393: Network error: Client connection negotiation failed: client connection to 127.7.168.124:43393: connect: Connection refused (error 111)
04:03:06.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:06.317894 8020 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.168.124" port: 43393 } attempt: 5
04:03:06.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:06.321446 8020 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.168.124:43393: Network error: Client connection negotiation failed: client connection to 127.7.168.124:43393: connect: Connection refused (error 111)
04:03:06.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:06.400771 7938 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.168.124" port: 43393 } attempt: 5
04:03:06.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:06.405269 7938 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.168.124:43393: Network error: Client connection negotiation failed: client connection to 127.7.168.124:43393: connect: Connection refused (error 111)
04:03:06.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:06.869060 8020 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.168.124" port: 43393 } attempt: 6
04:03:06.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:06.875790 8020 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.168.124:43393: Network error: Client connection negotiation failed: client connection to 127.7.168.124:43393: connect: Connection refused (error 111)
04:03:06.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:06.957680 7938 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.168.124" port: 43393 } attempt: 6
04:03:06.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:06.965171 7938 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.168.124:43393: Network error: Client connection negotiation failed: client connection to 127.7.168.124:43393: connect: Connection refused (error 111)
04:03:07.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:06.031047 8039 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:03:07.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:06.031620 8026 server_base.cc:1034] running on GCE node
04:03:07.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:06.032142 8036 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:03:07.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:07.266180 8026 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
04:03:07.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:07.268615 8026 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
04:03:07.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:07.270071 8026 hybrid_clock.cc:648] HybridClock initialized: now 1790568187270043 us; error 53 us; skew 500 ppm
04:03:07.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:07.270843 8026 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
04:03:07.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 04:03:07 dist-test-slave-jlk3 krb5kdc[7853](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790568187, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.168.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
04:03:07.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:07.276791 8026 init.cc:377] Logged in from keytab as kudu/127.7.168.124@KRBTEST.COM (short username kudu)
04:03:07.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:07.282608 8026 webserver.cc:460] Webserver started at http://127.7.168.124:39025/ using document root <none> and password file <none>
04:03:07.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:07.283576 8026 fs_manager.cc:359] Metadata directory not provided
04:03:07.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:07.283810 8026 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
04:03:07.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:07.284315 8026 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
04:03:07.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:07.289227 8026 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-2/data/instance:
04:03:07.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1fbf1cf69c8d451aaf2137cf567a6d7c"
04:03:07.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 04:03:07 on dist-test-slave-jlk3"
04:03:07.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:07.290555 8026 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-2/wal/instance:
04:03:07.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1fbf1cf69c8d451aaf2137cf567a6d7c"
04:03:07.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 04:03:07 on dist-test-slave-jlk3"
04:03:07.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:07.299319 8026 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.005s sys 0.004s
04:03:07.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:07.305048 8052 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
04:03:07.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:07.306128 8026 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
04:03:07.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:07.306469 8026 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-2/data,/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-2/wal
04:03:07.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1fbf1cf69c8d451aaf2137cf567a6d7c"
04:03:07.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 04:03:07 on dist-test-slave-jlk3"
04:03:07.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:07.306804 8026 fs_report.cc:383] FS layout report
04:03:07.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
04:03:07.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-2/wal
04:03:07.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-2/wal
04:03:07.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-2/data/data
04:03:07.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
04:03:07.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
04:03:07.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
04:03:07.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
04:03:07.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
04:03:07.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
04:03:07.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
04:03:07.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
04:03:07.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
04:03:07.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
04:03:07.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
04:03:07.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:07.345294 8026 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
04:03:07.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:07.350612 8026 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
04:03:07.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:07.351090 8026 kserver.cc:163] Server-wide thread pool size limit: 3276
04:03:07.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:07.428714 8026 rpc_server.cc:307] RPC server started. Bound to: 127.7.168.124:43393
04:03:07.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:07.428804 8103 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.168.124:43393 every 8 connection(s)
04:03:07.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:07.431486 8026 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-2/data/info.pb
04:03:07.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:07.436709 7841 external_mini_cluster.cc:1388] Started /tmp/dist-test-task4Qe23I/build/tsan/bin/kudu as pid 8026
04:03:07.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:07.437211 7841 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-2/wal/instance
04:03:07.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:07.437636 8104 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
04:03:07.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:07.454313 8104 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.168.126" port: 46569 } has no permanent_uuid. Determining permanent_uuid...
04:03:07.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 04:03:07 dist-test-slave-jlk3 krb5kdc[7853](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790568181, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.7.168.124@KRBTEST.COM
04:03:07.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 04:03:07 dist-test-slave-jlk3 krb5kdc[7853](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790568187, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.168.124@KRBTEST.COM for kudu/127.7.168.126@KRBTEST.COM
04:03:07.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:07.493161 8104 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.168.125" port: 46485 } has no permanent_uuid. Determining permanent_uuid...
04:03:07.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 04:03:07 dist-test-slave-jlk3 krb5kdc[7853](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790568187, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.168.124@KRBTEST.COM for kudu/127.7.168.125@KRBTEST.COM
04:03:07.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 04:03:07 dist-test-slave-jlk3 krb5kdc[7853](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790568181, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.7.168.125@KRBTEST.COM
04:03:07.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:07.520833 8104 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.168.124" port: 43393 } has no permanent_uuid. Determining permanent_uuid...
04:03:07.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 04:03:07 dist-test-slave-jlk3 krb5kdc[7853](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790568187, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.168.124@KRBTEST.COM for kudu/127.7.168.124@KRBTEST.COM
04:03:07.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 04:03:07 dist-test-slave-jlk3 krb5kdc[7853](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790568181, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.7.168.126@KRBTEST.COM
04:03:07.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:07.553040 7841 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
04:03:07.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:07.553087 8104 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c: Bootstrap starting.
04:03:07.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:07.558614 8104 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c: Neither blocks nor log segments found. Creating new log.
04:03:07.560 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
04:03:07.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:07.560266 8104 log.cc:824] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c: Log is configured to *not* fsync() on all Append() calls
04:03:07.562 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
04:03:07.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:07.565843 8104 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c: No bootstrap required, opened a new log
04:03:07.575 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
04:03:07.576 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
04:03:07.579 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.7.168.126"
port: 46569
}
bound_http_address {
host: "127.7.168.126"
port: 34837
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.7.168.125"
port: 46485
}
bound_http_address {
host: "127.7.168.125"
port: 42977
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.7.168.124"
port: 43393
}
bound_http_address {
host: "127.7.168.124"
port: 39025
}
}
}
04:03:07.580 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
04:03:07.580 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
04:03:07.580 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
04:03:07.583 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
04:03:07.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:07.595562 8104 raft_consensus.cc:348] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f67c27ee2e584d25850a0572b8a3b27c" member_type: VOTER last_known_addr { host: "127.7.168.126" port: 46569 } } peers { permanent_uuid: "455fed1171d64bdfa974a8e29d8c171c" member_type: VOTER last_known_addr { host: "127.7.168.125" port: 46485 } } peers { permanent_uuid: "1fbf1cf69c8d451aaf2137cf567a6d7c" member_type: VOTER last_known_addr { host: "127.7.168.124" port: 43393 } }
04:03:07.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:07.596517 8104 raft_consensus.cc:374] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
04:03:07.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:07.596849 8104 raft_consensus.cc:729] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1fbf1cf69c8d451aaf2137cf567a6d7c, State: Initialized, Role: FOLLOWER
04:03:07.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:07.597868 8104 consensus_queue.cc:260] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f67c27ee2e584d25850a0572b8a3b27c" member_type: VOTER last_known_addr { host: "127.7.168.126" port: 46569 } } peers { permanent_uuid: "455fed1171d64bdfa974a8e29d8c171c" member_type: VOTER last_known_addr { host: "127.7.168.125" port: 46485 } } peers { permanent_uuid: "1fbf1cf69c8d451aaf2137cf567a6d7c" member_type: VOTER last_known_addr { host: "127.7.168.124" port: 43393 } }
04:03:07.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:07.614379 8114 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f67c27ee2e584d25850a0572b8a3b27c" member_type: VOTER last_known_addr { host: "127.7.168.126" port: 46569 } } peers { permanent_uuid: "455fed1171d64bdfa974a8e29d8c171c" member_type: VOTER last_known_addr { host: "127.7.168.125" port: 46485 } } peers { permanent_uuid: "1fbf1cf69c8d451aaf2137cf567a6d7c" member_type: VOTER last_known_addr { host: "127.7.168.124" port: 43393 } } }
04:03:07.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:07.615187 8114 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [sys.catalog]: This master's current role is: FOLLOWER
04:03:07.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:07.616269 8104 sys_catalog.cc:564] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [sys.catalog]: configured and running, proceeding with master startup.
04:03:07.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 04:03:07 dist-test-slave-jlk3 krb5kdc[7853](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790568187, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
04:03:07.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
04:03:07.635 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
04:03:07.638 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/krb5kdc/krb5cc
04:03:07.646 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
04:03:07.647 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/krb5kdc/krb5cc
04:03:07.654 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
04:03:07.658 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
04:03:07.661 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
04:03:07.662 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
04:03:07.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:07.668111 8126 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:03:07.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:07.668430 8126 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:03:07.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:07.652297 8026 master_runner.cc:186] Error getting master registration for 127.7.168.126:46569: OK, instance_id { permanent_uuid: "f67c27ee2e584d25850a0572b8a3b27c" instance_seqno: 1790568183270734 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
04:03:07.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:07.690696 8026 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
04:03:07.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 04:03:07 dist-test-slave-jlk3 krb5kdc[7853](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790568187, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
04:03:07.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
04:03:07.708 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
04:03:07.708 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/krb5kdc/krb5cc
04:03:07.710 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
04:03:07.710 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.7.168.126:46569
04:03:07.712 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
04:03:07.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:07.712733 7841 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task4Qe23I/build/tsan/bin/kudu with pid 7868
04:03:07.736 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
04:03:07.736 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.7.168.125:46485
04:03:07.737 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
04:03:07.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:07.737504 7841 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task4Qe23I/build/tsan/bin/kudu with pid 7944
04:03:07.765 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
04:03:07.766 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.7.168.124:43393
04:03:07.766 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
04:03:07.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:07.766635 7841 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task4Qe23I/build/tsan/bin/kudu with pid 8026
04:03:07.785 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
04:03:07.786 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.7.168.126:46569
04:03:07.787 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
04:03:07.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:07.789119 7841 external_mini_cluster.cc:1326] Running /tmp/dist-test-task4Qe23I/build/tsan/bin/kudu
04:03:07.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4Qe23I/build/tsan/bin/kudu
04:03:07.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-0/wal
04:03:07.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-0/data
04:03:07.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
04:03:07.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
04:03:07.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
04:03:07.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:03:07.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:03:07.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:03:07.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-0/logs
04:03:07.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-0/data/info.pb
04:03:07.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:03:07.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
04:03:07.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
04:03:07.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
04:03:07.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
04:03:07.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:03:07.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:03:07.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:03:07.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
04:03:07.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
04:03:07.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:03:07.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:03:07.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.168.126:46569
04:03:07.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.168.126
04:03:07.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34837
04:03:07.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/krb5kdc/kudu_127.7.168.126.keytab
04:03:07.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.168.126
04:03:07.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
04:03:07.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
04:03:07.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
04:03:07.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.168.84:39921
04:03:07.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:03:07.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:03:07.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:03:07.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:03:07.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393
04:03:07.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:03:07.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34185/jwks.json
04:03:07.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
04:03:08.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:08.121234 8130 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
04:03:08.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:08.121872 8130 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
04:03:08.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:08.122459 8130 flags.cc:425] Enabled unsafe flag: --never_fsync=true
04:03:08.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:08.160930 8130 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
04:03:08.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:08.161156 8130 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
04:03:08.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:08.161430 8130 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
04:03:08.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:08.161621 8130 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
04:03:08.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:08.161804 8130 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
04:03:08.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:08.161998 8130 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
04:03:08.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:08.162130 8130 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34185/jwks.json
04:03:08.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:08.198835 8130 master_runner.cc:386] Master server non-default flags:
04:03:08.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:03:08.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.168.84:39921
04:03:08.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:03:08.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:03:08.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-0/data
04:03:08.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-0/wal
04:03:08.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
04:03:08.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:03:08.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393
04:03:08.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/krb5kdc/kudu_127.7.168.126.keytab
04:03:08.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.168.126
04:03:08.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:03:08.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:03:08.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:03:08.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.168.126:46569
04:03:08.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:03:08.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
04:03:08.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:03:08.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34185/jwks.json
04:03:08.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
04:03:08.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
04:03:08.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
04:03:08.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:03:08.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:03:08.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-0/data/info.pb
04:03:08.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.168.126
04:03:08.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34837
04:03:08.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
04:03:08.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:03:08.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
04:03:08.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
04:03:08.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:03:08.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-0/logs
04:03:08.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:03:08.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
04:03:08.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
04:03:08.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
04:03:08.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
04:03:08.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
04:03:08.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
04:03:08.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 02:50:13 UTC on ccycloud.quasar-purgqw.root.comops.site
04:03:08.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676527
04:03:08.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
04:03:08.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:08.199939 8130 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
04:03:08.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:08.200961 8130 file_cache.cc:493] Constructed file cache file cache with capacity 419430
04:03:08.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:08.217962 8138 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:03:09.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:08.218643 8137 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:03:09.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:08.221060 8130 server_base.cc:1034] running on GCE node
04:03:09.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:08.226136 8140 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:03:09.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:09.439564 8130 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
04:03:09.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:09.442368 8130 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
04:03:09.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:09.443790 8130 hybrid_clock.cc:648] HybridClock initialized: now 1790568189443758 us; error 35 us; skew 500 ppm
04:03:09.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:09.444798 8130 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
04:03:09.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 04:03:09 dist-test-slave-jlk3 krb5kdc[7853](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790568189, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.168.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
04:03:09.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:09.451093 8130 init.cc:377] Logged in from keytab as kudu/127.7.168.126@KRBTEST.COM (short username kudu)
04:03:09.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:09.456964 8130 webserver.cc:460] Webserver started at http://127.7.168.126:34837/ using document root <none> and password file <none>
04:03:09.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:09.458063 8130 fs_manager.cc:359] Metadata directory not provided
04:03:09.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:09.458328 8130 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
04:03:09.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:09.466985 8130 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.001s sys 0.004s
04:03:09.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:09.471472 8149 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
04:03:09.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:09.472604 8130 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
04:03:09.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:09.473150 8130 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-0/data,/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-0/wal
04:03:09.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f67c27ee2e584d25850a0572b8a3b27c"
04:03:09.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 04:03:03 on dist-test-slave-jlk3"
04:03:09.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:09.475728 8130 fs_report.cc:383] FS layout report
04:03:09.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
04:03:09.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-0/wal
04:03:09.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-0/wal
04:03:09.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-0/data/data
04:03:09.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
04:03:09.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
04:03:09.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
04:03:09.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
04:03:09.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
04:03:09.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
04:03:09.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
04:03:09.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
04:03:09.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
04:03:09.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
04:03:09.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
04:03:09.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:09.519443 8130 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
04:03:09.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:09.526417 8130 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
04:03:09.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:09.526990 8130 kserver.cc:163] Server-wide thread pool size limit: 3276
04:03:09.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:09.606500 8130 rpc_server.cc:307] RPC server started. Bound to: 127.7.168.126:46569
04:03:09.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:09.606621 8200 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.168.126:46569 every 8 connection(s)
04:03:09.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:09.609352 8130 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-0/data/info.pb
04:03:09.613 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
04:03:09.613 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.7.168.125:46485
04:03:09.614 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
04:03:09.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:09.612636 7841 external_mini_cluster.cc:1388] Started /tmp/dist-test-task4Qe23I/build/tsan/bin/kudu as pid 8130
04:03:09.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:09.615402 7841 external_mini_cluster.cc:1326] Running /tmp/dist-test-task4Qe23I/build/tsan/bin/kudu
04:03:09.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4Qe23I/build/tsan/bin/kudu
04:03:09.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-1/wal
04:03:09.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-1/data
04:03:09.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
04:03:09.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
04:03:09.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
04:03:09.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:03:09.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:03:09.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:03:09.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-1/logs
04:03:09.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-1/data/info.pb
04:03:09.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:03:09.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
04:03:09.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
04:03:09.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
04:03:09.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
04:03:09.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:03:09.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:03:09.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:03:09.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
04:03:09.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
04:03:09.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:03:09.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:03:09.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.168.125:46485
04:03:09.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.168.125
04:03:09.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42977
04:03:09.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/krb5kdc/kudu_127.7.168.125.keytab
04:03:09.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.168.125
04:03:09.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
04:03:09.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
04:03:09.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
04:03:09.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.168.84:39921
04:03:09.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:03:09.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:03:09.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:03:09.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:03:09.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393
04:03:09.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:03:09.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34185/jwks.json
04:03:09.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
04:03:09.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:09.622699 8201 sys_catalog.cc:263] Verifying existing consensus state
04:03:09.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260928 04:03:09.623625 8201 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
04:03:09.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:09.624099 8130 master.cc:537] Master@127.7.168.126:46569 shutting down...
04:03:09.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:09.683115 8130 master.cc:552] Master@127.7.168.126:46569 shutdown complete.
04:03:09.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
04:03:09.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:09.932109 8203 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
04:03:09.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:09.932638 8203 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
04:03:09.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:09.933280 8203 flags.cc:425] Enabled unsafe flag: --never_fsync=true
04:03:09.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:09.964327 8203 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
04:03:09.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:09.964612 8203 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
04:03:09.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:09.965059 8203 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
04:03:09.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:09.965415 8203 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
04:03:09.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:09.965668 8203 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
04:03:09.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:09.965860 8203 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
04:03:09.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:09.965996 8203 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34185/jwks.json
04:03:10.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:10.003862 8203 master_runner.cc:386] Master server non-default flags:
04:03:10.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:03:10.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.168.84:39921
04:03:10.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:03:10.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:03:10.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-1/data
04:03:10.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-1/wal
04:03:10.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
04:03:10.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:03:10.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393
04:03:10.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/krb5kdc/kudu_127.7.168.125.keytab
04:03:10.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.168.125
04:03:10.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:03:10.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:03:10.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:03:10.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.168.125:46485
04:03:10.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:03:10.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
04:03:10.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:03:10.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34185/jwks.json
04:03:10.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
04:03:10.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
04:03:10.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
04:03:10.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:03:10.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:03:10.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-1/data/info.pb
04:03:10.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.168.125
04:03:10.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42977
04:03:10.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
04:03:10.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:03:10.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
04:03:10.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
04:03:10.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:03:10.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-1/logs
04:03:10.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:03:10.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
04:03:10.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
04:03:10.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
04:03:10.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
04:03:10.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
04:03:10.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
04:03:10.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 02:50:13 UTC on ccycloud.quasar-purgqw.root.comops.site
04:03:10.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676527
04:03:10.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
04:03:10.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:10.005004 8203 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
04:03:10.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:10.006206 8203 file_cache.cc:493] Constructed file cache file cache with capacity 419430
04:03:10.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:10.024822 8209 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:03:11.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:10.024829 8212 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:03:11.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:10.025189 8210 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:03:11.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:11.261086 8211 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
04:03:11.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:11.261220 8203 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
04:03:11.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:11.265334 8203 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
04:03:11.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:11.268020 8203 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
04:03:11.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:11.269412 8203 hybrid_clock.cc:648] HybridClock initialized: now 1790568191269379 us; error 40 us; skew 500 ppm
04:03:11.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:11.270265 8203 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
04:03:11.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 04:03:11 dist-test-slave-jlk3 krb5kdc[7853](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790568191, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.168.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
04:03:11.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:11.283776 8203 init.cc:377] Logged in from keytab as kudu/127.7.168.125@KRBTEST.COM (short username kudu)
04:03:11.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:11.289307 8203 webserver.cc:460] Webserver started at http://127.7.168.125:42977/ using document root <none> and password file <none>
04:03:11.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:11.290365 8203 fs_manager.cc:359] Metadata directory not provided
04:03:11.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:11.290632 8203 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
04:03:11.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:11.298480 8203 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.001s sys 0.004s
04:03:11.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:11.302966 8222 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
04:03:11.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:11.304003 8203 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
04:03:11.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:11.304346 8203 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-1/data,/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-1/wal
04:03:11.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "455fed1171d64bdfa974a8e29d8c171c"
04:03:11.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 04:03:05 on dist-test-slave-jlk3"
04:03:11.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:11.306624 8203 fs_report.cc:383] FS layout report
04:03:11.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
04:03:11.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-1/wal
04:03:11.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-1/wal
04:03:11.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-1/data/data
04:03:11.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
04:03:11.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
04:03:11.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
04:03:11.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
04:03:11.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
04:03:11.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
04:03:11.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
04:03:11.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
04:03:11.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
04:03:11.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
04:03:11.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
04:03:11.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:11.345278 8203 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
04:03:11.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:11.350873 8203 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
04:03:11.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:11.351387 8203 kserver.cc:163] Server-wide thread pool size limit: 3276
04:03:11.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:11.423456 8203 rpc_server.cc:307] RPC server started. Bound to: 127.7.168.125:46485
04:03:11.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:11.423635 8273 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.168.125:46485 every 8 connection(s)
04:03:11.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:11.426666 8203 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-1/data/info.pb
04:03:11.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:11.430114 7841 external_mini_cluster.cc:1388] Started /tmp/dist-test-task4Qe23I/build/tsan/bin/kudu as pid 8203
04:03:11.431 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
04:03:11.431 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.7.168.124:43393
04:03:11.431 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
04:03:11.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:11.433151 7841 external_mini_cluster.cc:1326] Running /tmp/dist-test-task4Qe23I/build/tsan/bin/kudu
04:03:11.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4Qe23I/build/tsan/bin/kudu
04:03:11.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-2/wal
04:03:11.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-2/data
04:03:11.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
04:03:11.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
04:03:11.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
04:03:11.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:03:11.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:03:11.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:03:11.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-2/logs
04:03:11.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-2/data/info.pb
04:03:11.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:03:11.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
04:03:11.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
04:03:11.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
04:03:11.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
04:03:11.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:03:11.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:03:11.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:03:11.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
04:03:11.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
04:03:11.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:03:11.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:03:11.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.168.124:43393
04:03:11.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.168.124
04:03:11.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39025
04:03:11.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/krb5kdc/kudu_127.7.168.124.keytab
04:03:11.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.168.124
04:03:11.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
04:03:11.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
04:03:11.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
04:03:11.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.168.84:39921
04:03:11.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:03:11.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:03:11.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:03:11.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:03:11.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393
04:03:11.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:03:11.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34185/jwks.json
04:03:11.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
04:03:11.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:11.437681 8274 sys_catalog.cc:263] Verifying existing consensus state
04:03:11.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260928 04:03:11.438449 8274 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
04:03:11.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:11.438843 8203 master.cc:537] Master@127.7.168.125:46485 shutting down...
04:03:11.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:11.503012 8203 master.cc:552] Master@127.7.168.125:46485 shutdown complete.
04:03:11.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
04:03:11.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:11.754676 8276 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
04:03:11.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:11.755227 8276 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
04:03:11.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:11.755817 8276 flags.cc:425] Enabled unsafe flag: --never_fsync=true
04:03:11.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:11.788770 8276 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
04:03:11.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:11.789113 8276 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
04:03:11.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:11.789390 8276 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
04:03:11.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:11.789584 8276 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
04:03:11.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:11.789773 8276 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
04:03:11.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:11.789955 8276 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
04:03:11.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:11.790091 8276 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34185/jwks.json
04:03:11.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:11.834046 8276 master_runner.cc:386] Master server non-default flags:
04:03:11.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:03:11.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.168.84:39921
04:03:11.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:03:11.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:03:11.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-2/data
04:03:11.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-2/wal
04:03:11.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
04:03:11.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:03:11.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393
04:03:11.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/krb5kdc/kudu_127.7.168.124.keytab
04:03:11.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.168.124
04:03:11.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:03:11.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:03:11.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:03:11.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.168.124:43393
04:03:11.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:03:11.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
04:03:11.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:03:11.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34185/jwks.json
04:03:11.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
04:03:11.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
04:03:11.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
04:03:11.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:03:11.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:03:11.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-2/data/info.pb
04:03:11.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.168.124
04:03:11.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39025
04:03:11.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
04:03:11.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:03:11.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
04:03:11.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
04:03:11.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:03:11.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-2/logs
04:03:11.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:03:11.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
04:03:11.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
04:03:11.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
04:03:11.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
04:03:11.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
04:03:11.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
04:03:11.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 02:50:13 UTC on ccycloud.quasar-purgqw.root.comops.site
04:03:11.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676527
04:03:11.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
04:03:11.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:11.835150 8276 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
04:03:11.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:11.836064 8276 file_cache.cc:493] Constructed file cache file cache with capacity 419430
04:03:11.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:11.852618 8286 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:03:13.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:11.852713 8284 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:03:13.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:11.853552 8283 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:03:13.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:13.069720 8285 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
04:03:13.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:13.069859 8276 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
04:03:13.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:13.073971 8276 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
04:03:13.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:13.076632 8276 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
04:03:13.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:13.078082 8276 hybrid_clock.cc:648] HybridClock initialized: now 1790568193078055 us; error 42 us; skew 500 ppm
04:03:13.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:13.078940 8276 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
04:03:13.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 04:03:13 dist-test-slave-jlk3 krb5kdc[7853](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790568193, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.168.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
04:03:13.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:13.085837 8276 init.cc:377] Logged in from keytab as kudu/127.7.168.124@KRBTEST.COM (short username kudu)
04:03:13.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:13.091341 8276 webserver.cc:460] Webserver started at http://127.7.168.124:39025/ using document root <none> and password file <none>
04:03:13.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:13.092427 8276 fs_manager.cc:359] Metadata directory not provided
04:03:13.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:13.092706 8276 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
04:03:13.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:13.102557 8276 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.001s sys 0.003s
04:03:13.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:13.107735 8294 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
04:03:13.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:13.108896 8276 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
04:03:13.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:13.109241 8276 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-2/data,/tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-2/wal
04:03:13.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1fbf1cf69c8d451aaf2137cf567a6d7c"
04:03:13.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 04:03:07 on dist-test-slave-jlk3"
04:03:13.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:13.111728 8276 fs_report.cc:383] FS layout report
04:03:13.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
04:03:13.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-2/wal
04:03:13.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-2/wal
04:03:13.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-2/data/data
04:03:13.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
04:03:13.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
04:03:13.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
04:03:13.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
04:03:13.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
04:03:13.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
04:03:13.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
04:03:13.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
04:03:13.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
04:03:13.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
04:03:13.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
04:03:13.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:13.147646 8276 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
04:03:13.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:13.153443 8276 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
04:03:13.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:13.153853 8276 kserver.cc:163] Server-wide thread pool size limit: 3276
04:03:13.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:13.226217 8276 rpc_server.cc:307] RPC server started. Bound to: 127.7.168.124:43393
04:03:13.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:13.226413 8345 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.168.124:43393 every 8 connection(s)
04:03:13.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:13.229084 8276 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4Qe23I/test-tmp/mini-kudu-cluster7167086900816554256/master-2/data/info.pb
04:03:13.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:13.237314 7841 external_mini_cluster.cc:1388] Started /tmp/dist-test-task4Qe23I/build/tsan/bin/kudu as pid 8276
04:03:13.237 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
04:03:13.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:13.242442 8346 sys_catalog.cc:263] Verifying existing consensus state
04:03:13.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:13.248502 8346 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c: Bootstrap starting.
04:03:13.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:13.292564 8346 log.cc:824] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c: Log is configured to *not* fsync() on all Append() calls
04:03:13.295 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.7.168.126:46569(127.7.168.126:46569)] connecting to peer
04:03:13.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:13.297925 8346 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
04:03:13.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:13.298794 8346 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c: Bootstrap complete.
04:03:13.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:13.334746 8346 raft_consensus.cc:348] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f67c27ee2e584d25850a0572b8a3b27c" member_type: VOTER last_known_addr { host: "127.7.168.126" port: 46569 } } peers { permanent_uuid: "455fed1171d64bdfa974a8e29d8c171c" member_type: VOTER last_known_addr { host: "127.7.168.125" port: 46485 } } peers { permanent_uuid: "1fbf1cf69c8d451aaf2137cf567a6d7c" member_type: VOTER last_known_addr { host: "127.7.168.124" port: 43393 } }
04:03:13.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:13.338088 8346 raft_consensus.cc:374] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
04:03:13.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:13.338400 8346 raft_consensus.cc:729] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1fbf1cf69c8d451aaf2137cf567a6d7c, State: Initialized, Role: FOLLOWER
04:03:13.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:13.339421 8346 consensus_queue.cc:260] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f67c27ee2e584d25850a0572b8a3b27c" member_type: VOTER last_known_addr { host: "127.7.168.126" port: 46569 } } peers { permanent_uuid: "455fed1171d64bdfa974a8e29d8c171c" member_type: VOTER last_known_addr { host: "127.7.168.125" port: 46485 } } peers { permanent_uuid: "1fbf1cf69c8d451aaf2137cf567a6d7c" member_type: VOTER last_known_addr { host: "127.7.168.124" port: 43393 } }
04:03:13.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:13.344772 8350 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f67c27ee2e584d25850a0572b8a3b27c" member_type: VOTER last_known_addr { host: "127.7.168.126" port: 46569 } } peers { permanent_uuid: "455fed1171d64bdfa974a8e29d8c171c" member_type: VOTER last_known_addr { host: "127.7.168.125" port: 46485 } } peers { permanent_uuid: "1fbf1cf69c8d451aaf2137cf567a6d7c" member_type: VOTER last_known_addr { host: "127.7.168.124" port: 43393 } } }
04:03:13.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:13.345731 8350 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [sys.catalog]: This master's current role is: FOLLOWER
04:03:13.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:13.346575 8346 sys_catalog.cc:564] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [sys.catalog]: configured and running, proceeding with master startup.
04:03:13.348 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.7.168.125:46485(127.7.168.125:46485)] connecting to peer
04:03:13.350 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.7.168.124:43393(127.7.168.124:43393)] connecting to peer
04:03:13.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:13.409035 8300 tablet.cc:2369] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c: Can't schedule compaction. Clean time has not been advanced past its initial value.
04:03:13.410 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:13.411 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.168.125:46485(127.7.168.125:46485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:13.413 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:13.413 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.168.126:46569(127.7.168.126:46569)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:13.415 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.168.126:46569: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:13.417 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.7.168.124:43393(127.7.168.124:43393)] Successfully connected to peer
04:03:13.418 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.168.125:46485: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:13.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:13.381963 8276 master_runner.cc:186] Error getting master registration for 127.7.168.126:46569: Network error: Client connection negotiation failed: client connection to 127.7.168.126:46569: connect: Connection refused (error 111),
04:03:13.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:13.425190 8276 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
04:03:13.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:13.433058 8363 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:03:13.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:13.433403 8363 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:03:13.516 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
04:03:13.525 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.7.168.124
04:03:13.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 04:03:13 dist-test-slave-jlk3 krb5kdc[7853](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790568187, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.7.168.124@KRBTEST.COM
04:03:13.786 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x03c491fa, L:/127.0.0.1:48116 - R:/127.7.168.124:43393] using SASL/GSSAPI
04:03:13.803 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393; will retry
04:03:13.823 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.168.126:46569(127.7.168.126:46569)] connecting to peer
04:03:13.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:13.823786 8350 raft_consensus.cc:482] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
04:03:13.825 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.168.125:46485(127.7.168.125:46485)] connecting to peer
04:03:13.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:13.824307 8350 raft_consensus.cc:504] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f67c27ee2e584d25850a0572b8a3b27c" member_type: VOTER last_known_addr { host: "127.7.168.126" port: 46569 } } peers { permanent_uuid: "455fed1171d64bdfa974a8e29d8c171c" member_type: VOTER last_known_addr { host: "127.7.168.125" port: 46485 } } peers { permanent_uuid: "1fbf1cf69c8d451aaf2137cf567a6d7c" member_type: VOTER last_known_addr { host: "127.7.168.124" port: 43393 } }
04:03:13.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:13.828333 8350 leader_election.cc:288] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f67c27ee2e584d25850a0572b8a3b27c (127.7.168.126:46569), 455fed1171d64bdfa974a8e29d8c171c (127.7.168.125:46485)
04:03:13.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:13.830297 8298 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.168.126:46569: connect: Connection refused (error 111)
04:03:13.832 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:13.832 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.168.126:46569(127.7.168.126:46569)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:13.833 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.168.126:46569: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:13.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:13.834163 8296 leader_election.cc:334] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 455fed1171d64bdfa974a8e29d8c171c (127.7.168.125:46485): Network error: Client connection negotiation failed: client connection to 127.7.168.125:46485: connect: Connection refused (error 111)
04:03:13.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:13.837117 8298 leader_election.cc:334] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer f67c27ee2e584d25850a0572b8a3b27c (127.7.168.126:46569): Network error: Client connection negotiation failed: client connection to 127.7.168.126:46569: connect: Connection refused (error 111)
04:03:13.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:13.837500 8298 leader_election.cc:302] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1fbf1cf69c8d451aaf2137cf567a6d7c; no voters: 455fed1171d64bdfa974a8e29d8c171c, f67c27ee2e584d25850a0572b8a3b27c
04:03:13.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:13.838403 8350 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
04:03:13.842 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:13.842 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.168.125:46485(127.7.168.125:46485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:13.843 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.168.125:46485: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:13.844 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393; will retry
04:03:13.862 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.168.126:46569(127.7.168.126:46569)] connecting to peer
04:03:13.864 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.168.125:46485(127.7.168.125:46485)] connecting to peer
04:03:13.864 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:13.865 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.168.126:46569(127.7.168.126:46569)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:13.865 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.168.126:46569: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:13.868 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:13.868 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.168.125:46485(127.7.168.125:46485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:13.869 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.168.125:46485: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:13.871 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393; will retry
04:03:13.882 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.168.126:46569(127.7.168.126:46569)] connecting to peer
04:03:13.883 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.168.125:46485(127.7.168.125:46485)] connecting to peer
04:03:13.884 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:13.884 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.168.126:46569(127.7.168.126:46569)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:13.885 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.168.126:46569: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:13.886 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:13.886 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.168.125:46485(127.7.168.125:46485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:13.887 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.168.125:46485: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:13.890 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393; will retry
04:03:13.923 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.168.126:46569(127.7.168.126:46569)] connecting to peer
04:03:13.924 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.168.125:46485(127.7.168.125:46485)] connecting to peer
04:03:13.925 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:13.925 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.168.126:46569(127.7.168.126:46569)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:13.926 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.168.126:46569: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:13.928 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:13.928 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.168.125:46485(127.7.168.125:46485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:13.929 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.168.125:46485: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:13.931 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393; will retry
04:03:13.963 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.168.126:46569(127.7.168.126:46569)] connecting to peer
04:03:13.964 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.168.125:46485(127.7.168.125:46485)] connecting to peer
04:03:13.965 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:13.965 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.168.126:46569(127.7.168.126:46569)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:13.966 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.168.126:46569: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:13.968 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:13.969 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.168.125:46485(127.7.168.125:46485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:13.971 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.168.125:46485: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:13.973 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393; will retry
04:03:14.043 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.168.126:46569(127.7.168.126:46569)] connecting to peer
04:03:14.045 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.168.125:46485(127.7.168.125:46485)] connecting to peer
04:03:14.047 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:14.047 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.168.126:46569(127.7.168.126:46569)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:14.048 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.168.126:46569: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:14.049 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:14.049 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.168.125:46485(127.7.168.125:46485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:14.050 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.168.125:46485: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:14.052 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393; will retry
04:03:14.182 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.168.126:46569(127.7.168.126:46569)] connecting to peer
04:03:14.184 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.168.125:46485(127.7.168.125:46485)] connecting to peer
04:03:14.185 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:14.186 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.168.126:46569(127.7.168.126:46569)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:14.186 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:14.187 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.168.125:46485(127.7.168.125:46485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:14.187 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.168.126:46569: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:14.188 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.168.125:46485: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:14.192 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393; will retry
04:03:14.382 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.168.126:46569(127.7.168.126:46569)] connecting to peer
04:03:14.383 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.168.125:46485(127.7.168.125:46485)] connecting to peer
04:03:14.385 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:14.385 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.168.126:46569(127.7.168.126:46569)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:14.386 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.168.126:46569: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:14.386 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:14.386 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.168.125:46485(127.7.168.125:46485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:14.387 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.168.125:46485: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:14.389 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393; will retry
04:03:14.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:14.436324 8363 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:03:14.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:14.436712 8363 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:03:14.902 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.168.126:46569(127.7.168.126:46569)] connecting to peer
04:03:14.904 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.168.125:46485(127.7.168.125:46485)] connecting to peer
04:03:14.905 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:14.905 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.168.126:46569(127.7.168.126:46569)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:14.905 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.168.126:46569: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:14.907 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:14.908 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.168.125:46485(127.7.168.125:46485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:14.909 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.168.125:46485: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:14.911 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393; will retry
04:03:15.142 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.168.126:46569(127.7.168.126:46569)] connecting to peer
04:03:15.143 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.168.125:46485(127.7.168.125:46485)] connecting to peer
04:03:15.144 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:15.144 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.168.126:46569(127.7.168.126:46569)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:15.145 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.168.126:46569: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:15.146 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:15.146 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.168.125:46485(127.7.168.125:46485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:15.146 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.168.125:46485: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:15.148 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393; will retry
04:03:15.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:15.439426 8363 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:03:15.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:15.439705 8363 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:03:15.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:15.650652 8371 raft_consensus.cc:482] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
04:03:15.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:15.651006 8371 raft_consensus.cc:504] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f67c27ee2e584d25850a0572b8a3b27c" member_type: VOTER last_known_addr { host: "127.7.168.126" port: 46569 } } peers { permanent_uuid: "455fed1171d64bdfa974a8e29d8c171c" member_type: VOTER last_known_addr { host: "127.7.168.125" port: 46485 } } peers { permanent_uuid: "1fbf1cf69c8d451aaf2137cf567a6d7c" member_type: VOTER last_known_addr { host: "127.7.168.124" port: 43393 } }
04:03:15.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:15.652393 8371 leader_election.cc:288] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f67c27ee2e584d25850a0572b8a3b27c (127.7.168.126:46569), 455fed1171d64bdfa974a8e29d8c171c (127.7.168.125:46485)
04:03:15.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:15.658077 8298 leader_election.cc:334] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer f67c27ee2e584d25850a0572b8a3b27c (127.7.168.126:46569): Network error: Client connection negotiation failed: client connection to 127.7.168.126:46569: connect: Connection refused (error 111)
04:03:15.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:15.658433 8296 leader_election.cc:334] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 455fed1171d64bdfa974a8e29d8c171c (127.7.168.125:46485): Network error: Client connection negotiation failed: client connection to 127.7.168.125:46485: connect: Connection refused (error 111)
04:03:15.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:15.658772 8296 leader_election.cc:302] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1fbf1cf69c8d451aaf2137cf567a6d7c; no voters: 455fed1171d64bdfa974a8e29d8c171c, f67c27ee2e584d25850a0572b8a3b27c
04:03:15.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:15.659278 8371 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
04:03:15.922 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.168.126:46569(127.7.168.126:46569)] connecting to peer
04:03:15.924 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.168.125:46485(127.7.168.125:46485)] connecting to peer
04:03:15.925 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:15.925 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.168.126:46569(127.7.168.126:46569)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:15.926 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.168.126:46569: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:15.926 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:15.927 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.168.125:46485(127.7.168.125:46485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:15.927 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.168.125:46485: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:15.929 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393; will retry
04:03:16.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:16.442668 8363 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:03:16.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:16.442970 8363 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:03:16.482 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.168.126:46569(127.7.168.126:46569)] connecting to peer
04:03:16.483 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.168.125:46485(127.7.168.125:46485)] connecting to peer
04:03:16.484 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:16.484 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.168.126:46569(127.7.168.126:46569)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:16.484 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.168.126:46569: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:16.485 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:16.485 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.168.125:46485(127.7.168.125:46485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:16.485 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.168.125:46485: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:16.487 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393; will retry
04:03:17.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:17.445689 8363 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:03:17.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:17.445968 8363 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:03:18.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:18.428288 8276 master_runner.cc:186] Error getting master registration for 127.7.168.126:46569: Network error: Client connection negotiation failed: client connection to 127.7.168.126:46569: connect: Connection refused (error 111),
04:03:18.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:18.428746 8276 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
04:03:18.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:18.449198 8363 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:03:18.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:18.449527 8363 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:03:18.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:18.879956 8380 raft_consensus.cc:482] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
04:03:18.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:18.880362 8380 raft_consensus.cc:504] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f67c27ee2e584d25850a0572b8a3b27c" member_type: VOTER last_known_addr { host: "127.7.168.126" port: 46569 } } peers { permanent_uuid: "455fed1171d64bdfa974a8e29d8c171c" member_type: VOTER last_known_addr { host: "127.7.168.125" port: 46485 } } peers { permanent_uuid: "1fbf1cf69c8d451aaf2137cf567a6d7c" member_type: VOTER last_known_addr { host: "127.7.168.124" port: 43393 } }
04:03:18.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:18.882181 8380 leader_election.cc:288] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f67c27ee2e584d25850a0572b8a3b27c (127.7.168.126:46569), 455fed1171d64bdfa974a8e29d8c171c (127.7.168.125:46485)
04:03:18.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:18.883239 8298 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.168.126:46569: connect: Connection refused (error 111) [suppressed 3 similar messages]
04:03:18.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:18.886927 8298 leader_election.cc:334] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer f67c27ee2e584d25850a0572b8a3b27c (127.7.168.126:46569): Network error: Client connection negotiation failed: client connection to 127.7.168.126:46569: connect: Connection refused (error 111)
04:03:18.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:18.887367 8296 leader_election.cc:334] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 455fed1171d64bdfa974a8e29d8c171c (127.7.168.125:46485): Network error: Client connection negotiation failed: client connection to 127.7.168.125:46485: connect: Connection refused (error 111)
04:03:18.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:18.887761 8296 leader_election.cc:302] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1fbf1cf69c8d451aaf2137cf567a6d7c; no voters: 455fed1171d64bdfa974a8e29d8c171c, f67c27ee2e584d25850a0572b8a3b27c
04:03:18.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:18.888320 8380 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
04:03:19.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:19.452399 8363 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:03:19.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:19.452816 8363 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:03:19.522 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.168.126:46569(127.7.168.126:46569)] connecting to peer
04:03:19.523 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.168.125:46485(127.7.168.125:46485)] connecting to peer
04:03:19.524 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:19.524 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.168.126:46569(127.7.168.126:46569)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:19.525 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.168.126:46569: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:19.526 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:19.526 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.168.125:46485(127.7.168.125:46485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:19.526 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.168.125:46485: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:19.528 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393; will retry
04:03:20.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:20.456730 8363 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:03:20.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:20.457034 8363 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:03:21.262 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.168.126:46569(127.7.168.126:46569)] connecting to peer
04:03:21.263 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.168.125:46485(127.7.168.125:46485)] connecting to peer
04:03:21.264 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:21.265 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.168.126:46569(127.7.168.126:46569)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:21.265 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.168.126:46569: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:21.266 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:21.266 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.168.125:46485(127.7.168.125:46485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:21.266 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.168.125:46485: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:21.268 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393; will retry
04:03:21.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:21.459872 8363 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:03:21.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:21.460175 8363 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:03:22.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:22.464181 8363 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:03:22.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:22.464597 8363 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:03:22.542 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.168.126:46569(127.7.168.126:46569)] connecting to peer
04:03:22.544 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.168.125:46485(127.7.168.125:46485)] connecting to peer
04:03:22.545 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:22.545 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.168.126:46569(127.7.168.126:46569)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:22.545 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.168.126:46569: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:22.546 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:22.546 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.168.125:46485(127.7.168.125:46485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:22.547 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.168.125:46485: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:22.549 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393; will retry
04:03:23.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 04:03:23 dist-test-slave-jlk3 krb5kdc[7853](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790568203, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.168.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
04:03:23.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:23.089841 8290 init.cc:295] Successfully reacquired a new kerberos TGT
04:03:23.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:23.430711 8386 raft_consensus.cc:482] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
04:03:23.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:23.431391 8386 raft_consensus.cc:504] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f67c27ee2e584d25850a0572b8a3b27c" member_type: VOTER last_known_addr { host: "127.7.168.126" port: 46569 } } peers { permanent_uuid: "455fed1171d64bdfa974a8e29d8c171c" member_type: VOTER last_known_addr { host: "127.7.168.125" port: 46485 } } peers { permanent_uuid: "1fbf1cf69c8d451aaf2137cf567a6d7c" member_type: VOTER last_known_addr { host: "127.7.168.124" port: 43393 } }
04:03:23.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:23.433125 8276 master_runner.cc:186] Error getting master registration for 127.7.168.126:46569: Network error: Client connection negotiation failed: client connection to 127.7.168.126:46569: connect: Connection refused (error 111),
04:03:23.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:23.433405 8386 leader_election.cc:288] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f67c27ee2e584d25850a0572b8a3b27c (127.7.168.126:46569), 455fed1171d64bdfa974a8e29d8c171c (127.7.168.125:46485)
04:03:23.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:23.433693 8276 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
04:03:23.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:23.437500 8298 leader_election.cc:334] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer f67c27ee2e584d25850a0572b8a3b27c (127.7.168.126:46569): Network error: Client connection negotiation failed: client connection to 127.7.168.126:46569: connect: Connection refused (error 111)
04:03:23.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:23.438082 8296 leader_election.cc:334] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 455fed1171d64bdfa974a8e29d8c171c (127.7.168.125:46485): Network error: Client connection negotiation failed: client connection to 127.7.168.125:46485: connect: Connection refused (error 111)
04:03:23.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:23.438419 8296 leader_election.cc:302] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1fbf1cf69c8d451aaf2137cf567a6d7c; no voters: 455fed1171d64bdfa974a8e29d8c171c, f67c27ee2e584d25850a0572b8a3b27c
04:03:23.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:23.439020 8386 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
04:03:23.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:23.467418 8363 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:03:23.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:23.467717 8363 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:03:23.602 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.168.126:46569(127.7.168.126:46569)] connecting to peer
04:03:23.604 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.168.125:46485(127.7.168.125:46485)] connecting to peer
04:03:23.605 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:23.605 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.168.126:46569(127.7.168.126:46569)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:23.606 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.168.126:46569: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:23.606 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:23.606 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.168.125:46485(127.7.168.125:46485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:23.607 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.168.125:46485: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:23.609 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393; will retry
04:03:24.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:24.470422 8363 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:03:24.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:24.470706 8363 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:03:25.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:25.473490 8363 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:03:25.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:25.473838 8363 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:03:26.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:26.476504 8363 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:03:26.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:26.476821 8363 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:03:26.603 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.168.126:46569(127.7.168.126:46569)] connecting to peer
04:03:26.604 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.168.125:46485(127.7.168.125:46485)] connecting to peer
04:03:26.605 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:26.606 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.168.126:46569(127.7.168.126:46569)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:26.606 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.168.126:46569: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:26.607 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:26.607 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.168.125:46485(127.7.168.125:46485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:26.607 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.168.125:46485: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:26.609 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393; will retry
04:03:27.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:27.479436 8363 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:03:27.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:27.479686 8363 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:03:28.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:28.085929 8395 raft_consensus.cc:482] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
04:03:28.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:28.086702 8395 raft_consensus.cc:504] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f67c27ee2e584d25850a0572b8a3b27c" member_type: VOTER last_known_addr { host: "127.7.168.126" port: 46569 } } peers { permanent_uuid: "455fed1171d64bdfa974a8e29d8c171c" member_type: VOTER last_known_addr { host: "127.7.168.125" port: 46485 } } peers { permanent_uuid: "1fbf1cf69c8d451aaf2137cf567a6d7c" member_type: VOTER last_known_addr { host: "127.7.168.124" port: 43393 } }
04:03:28.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:28.088838 8395 leader_election.cc:288] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f67c27ee2e584d25850a0572b8a3b27c (127.7.168.126:46569), 455fed1171d64bdfa974a8e29d8c171c (127.7.168.125:46485)
04:03:28.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:28.090981 8298 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.168.126:46569: connect: Connection refused (error 111) [suppressed 3 similar messages]
04:03:28.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:28.094022 8298 leader_election.cc:334] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer f67c27ee2e584d25850a0572b8a3b27c (127.7.168.126:46569): Network error: Client connection negotiation failed: client connection to 127.7.168.126:46569: connect: Connection refused (error 111)
04:03:28.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:28.094436 8296 leader_election.cc:334] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 455fed1171d64bdfa974a8e29d8c171c (127.7.168.125:46485): Network error: Client connection negotiation failed: client connection to 127.7.168.125:46485: connect: Connection refused (error 111)
04:03:28.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:28.094825 8296 leader_election.cc:302] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1fbf1cf69c8d451aaf2137cf567a6d7c; no voters: 455fed1171d64bdfa974a8e29d8c171c, f67c27ee2e584d25850a0572b8a3b27c
04:03:28.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:28.095932 8395 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
04:03:28.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:28.435860 8276 master_runner.cc:186] Error getting master registration for 127.7.168.126:46569: Network error: Client connection negotiation failed: client connection to 127.7.168.126:46569: connect: Connection refused (error 111),
04:03:28.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:28.436316 8276 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
04:03:28.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:28.482820 8363 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:03:28.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:28.483070 8363 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:03:29.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:29.485671 8363 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:03:29.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:29.485921 8363 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:03:29.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:29.760083 8400 raft_consensus.cc:482] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
04:03:29.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:29.760456 8400 raft_consensus.cc:504] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f67c27ee2e584d25850a0572b8a3b27c" member_type: VOTER last_known_addr { host: "127.7.168.126" port: 46569 } } peers { permanent_uuid: "455fed1171d64bdfa974a8e29d8c171c" member_type: VOTER last_known_addr { host: "127.7.168.125" port: 46485 } } peers { permanent_uuid: "1fbf1cf69c8d451aaf2137cf567a6d7c" member_type: VOTER last_known_addr { host: "127.7.168.124" port: 43393 } }
04:03:29.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:29.762111 8400 leader_election.cc:288] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f67c27ee2e584d25850a0572b8a3b27c (127.7.168.126:46569), 455fed1171d64bdfa974a8e29d8c171c (127.7.168.125:46485)
04:03:29.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:29.766508 8298 leader_election.cc:334] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer f67c27ee2e584d25850a0572b8a3b27c (127.7.168.126:46569): Network error: Client connection negotiation failed: client connection to 127.7.168.126:46569: connect: Connection refused (error 111)
04:03:29.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:29.766873 8296 leader_election.cc:334] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 455fed1171d64bdfa974a8e29d8c171c (127.7.168.125:46485): Network error: Client connection negotiation failed: client connection to 127.7.168.125:46485: connect: Connection refused (error 111)
04:03:29.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:29.767167 8296 leader_election.cc:302] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1fbf1cf69c8d451aaf2137cf567a6d7c; no voters: 455fed1171d64bdfa974a8e29d8c171c, f67c27ee2e584d25850a0572b8a3b27c
04:03:29.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:29.767639 8400 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
04:03:30.202 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.168.126:46569(127.7.168.126:46569)] connecting to peer
04:03:30.203 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.168.125:46485(127.7.168.125:46485)] connecting to peer
04:03:30.204 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:30.205 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.168.126:46569(127.7.168.126:46569)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:30.205 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.168.126:46569: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:30.206 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:30.206 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.168.125:46485(127.7.168.125:46485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:30.206 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.168.125:46485: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:30.208 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393; will retry
04:03:30.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:30.488503 8363 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:03:30.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:30.488806 8363 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:03:31.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:31.491588 8363 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:03:31.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:31.491822 8363 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:03:32.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:32.495393 8363 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:03:32.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:32.495700 8363 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:03:32.982 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.168.126:46569(127.7.168.126:46569)] connecting to peer
04:03:32.984 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.168.125:46485(127.7.168.125:46485)] connecting to peer
04:03:32.985 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:32.985 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.168.126:46569(127.7.168.126:46569)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:32.985 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.168.126:46569: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:32.986 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:32.986 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.168.125:46485(127.7.168.125:46485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:32.986 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.168.125:46485: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:32.989 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393; will retry
04:03:33.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 04:03:33 dist-test-slave-jlk3 krb5kdc[7853](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790568213, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.168.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
04:03:33.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:33.092337 8290 init.cc:295] Successfully reacquired a new kerberos TGT
04:03:33.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:33.439347 8276 master_runner.cc:186] Error getting master registration for 127.7.168.126:46569: Network error: Client connection negotiation failed: client connection to 127.7.168.126:46569: connect: Connection refused (error 111),
04:03:33.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:33.439803 8276 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
04:03:33.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:33.498498 8363 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:03:33.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:33.498769 8363 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:03:33.803 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.168.126:46569(127.7.168.126:46569)] connecting to peer
04:03:33.803 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.168.125:46485(127.7.168.125:46485)] connecting to peer
04:03:33.804 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:33.804 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.168.126:46569(127.7.168.126:46569)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:33.804 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.168.126:46569: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:33.805 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:33.805 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.168.125:46485(127.7.168.125:46485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:33.805 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.168.125:46485: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:33.808 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393; will retry
04:03:34.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:34.501367 8363 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:03:34.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:34.501632 8363 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:03:35.202 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.168.126:46569(127.7.168.126:46569)] connecting to peer
04:03:35.203 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.168.125:46485(127.7.168.125:46485)] connecting to peer
04:03:35.204 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:35.204 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.168.126:46569(127.7.168.126:46569)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:35.204 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.168.126:46569: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:35.205 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:35.206 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.168.125:46485(127.7.168.125:46485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:35.206 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.168.125:46485: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:35.208 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393; will retry
04:03:35.302 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.168.126:46569(127.7.168.126:46569)] connecting to peer
04:03:35.303 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.168.125:46485(127.7.168.125:46485)] connecting to peer
04:03:35.303 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:35.303 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.168.126:46569(127.7.168.126:46569)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:35.303 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.168.126:46569: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:35.304 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:35.304 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.168.125:46485(127.7.168.125:46485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:35.304 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.168.125:46485: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:35.306 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393; will retry
04:03:35.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:35.498832 8411 raft_consensus.cc:482] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
04:03:35.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:35.499241 8411 raft_consensus.cc:504] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f67c27ee2e584d25850a0572b8a3b27c" member_type: VOTER last_known_addr { host: "127.7.168.126" port: 46569 } } peers { permanent_uuid: "455fed1171d64bdfa974a8e29d8c171c" member_type: VOTER last_known_addr { host: "127.7.168.125" port: 46485 } } peers { permanent_uuid: "1fbf1cf69c8d451aaf2137cf567a6d7c" member_type: VOTER last_known_addr { host: "127.7.168.124" port: 43393 } }
04:03:35.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:35.501036 8411 leader_election.cc:288] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f67c27ee2e584d25850a0572b8a3b27c (127.7.168.126:46569), 455fed1171d64bdfa974a8e29d8c171c (127.7.168.125:46485)
04:03:35.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:35.502856 8298 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.168.126:46569: connect: Connection refused (error 111) [suppressed 3 similar messages]
04:03:35.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:35.505452 8363 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:03:35.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:35.505729 8363 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:03:35.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:35.506090 8298 leader_election.cc:334] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer f67c27ee2e584d25850a0572b8a3b27c (127.7.168.126:46569): Network error: Client connection negotiation failed: client connection to 127.7.168.126:46569: connect: Connection refused (error 111)
04:03:35.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:35.506673 8296 leader_election.cc:334] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 455fed1171d64bdfa974a8e29d8c171c (127.7.168.125:46485): Network error: Client connection negotiation failed: client connection to 127.7.168.125:46485: connect: Connection refused (error 111)
04:03:35.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:35.506933 8296 leader_election.cc:302] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1fbf1cf69c8d451aaf2137cf567a6d7c; no voters: 455fed1171d64bdfa974a8e29d8c171c, f67c27ee2e584d25850a0572b8a3b27c
04:03:35.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:35.507436 8411 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
04:03:36.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:36.267372 7826 debug-util.cc:398] Leaking SignalData structure 0x7b08000bbb80 after lost signal to thread 7701
04:03:36.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:36.268301 7826 debug-util.cc:398] Leaking SignalData structure 0x7b08000af940 after lost signal to thread 7829
04:03:36.322 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.168.126:46569(127.7.168.126:46569)] connecting to peer
04:03:36.323 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.168.125:46485(127.7.168.125:46485)] connecting to peer
04:03:36.323 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:36.323 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.168.126:46569(127.7.168.126:46569)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:36.323 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.168.126:46569: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:36.324 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:36.325 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.168.125:46485(127.7.168.125:46485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:36.325 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.168.125:46485: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:36.327 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393; will retry
04:03:36.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:36.508450 8363 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:03:36.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:36.508810 8363 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:03:37.482 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.168.126:46569(127.7.168.126:46569)] connecting to peer
04:03:37.483 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.168.125:46485(127.7.168.125:46485)] connecting to peer
04:03:37.483 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:37.484 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.168.126:46569(127.7.168.126:46569)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:37.484 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.168.126:46569: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:37.485 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:37.485 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.168.125:46485(127.7.168.125:46485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:37.485 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.168.125:46485: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:37.487 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393; will retry
04:03:37.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:37.511489 8363 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:03:37.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:37.511734 8363 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:03:38.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:38.442371 8276 master_runner.cc:186] Error getting master registration for 127.7.168.126:46569: Network error: Client connection negotiation failed: client connection to 127.7.168.126:46569: connect: Connection refused (error 111),
04:03:38.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:38.442766 8276 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
04:03:38.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:38.514503 8363 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:03:38.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:38.514755 8363 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:03:39.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:39.517532 8363 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:03:39.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:39.517781 8363 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:03:39.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:39.863667 8421 raft_consensus.cc:482] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
04:03:39.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:39.864075 8421 raft_consensus.cc:504] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f67c27ee2e584d25850a0572b8a3b27c" member_type: VOTER last_known_addr { host: "127.7.168.126" port: 46569 } } peers { permanent_uuid: "455fed1171d64bdfa974a8e29d8c171c" member_type: VOTER last_known_addr { host: "127.7.168.125" port: 46485 } } peers { permanent_uuid: "1fbf1cf69c8d451aaf2137cf567a6d7c" member_type: VOTER last_known_addr { host: "127.7.168.124" port: 43393 } }
04:03:39.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:39.866081 8421 leader_election.cc:288] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f67c27ee2e584d25850a0572b8a3b27c (127.7.168.126:46569), 455fed1171d64bdfa974a8e29d8c171c (127.7.168.125:46485)
04:03:39.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:39.870216 8298 leader_election.cc:334] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer f67c27ee2e584d25850a0572b8a3b27c (127.7.168.126:46569): Network error: Client connection negotiation failed: client connection to 127.7.168.126:46569: connect: Connection refused (error 111)
04:03:39.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:39.870704 8296 leader_election.cc:334] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 455fed1171d64bdfa974a8e29d8c171c (127.7.168.125:46485): Network error: Client connection negotiation failed: client connection to 127.7.168.125:46485: connect: Connection refused (error 111)
04:03:39.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:39.870983 8296 leader_election.cc:302] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1fbf1cf69c8d451aaf2137cf567a6d7c; no voters: 455fed1171d64bdfa974a8e29d8c171c, f67c27ee2e584d25850a0572b8a3b27c
04:03:39.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:39.871536 8421 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 1fbf1cf69c8d451aaf2137cf567a6d7c [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
04:03:39.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:39.928526 7189 debug-util.cc:398] Leaking SignalData structure 0x7b08000b9f60 after lost signal to thread 7126
04:03:39.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:39.929242 7189 debug-util.cc:398] Leaking SignalData structure 0x7b0800081860 after lost signal to thread 7192
04:03:40.363 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.168.126:46569(127.7.168.126:46569)] connecting to peer
04:03:40.363 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.168.125:46485(127.7.168.125:46485)] connecting to peer
04:03:40.364 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:40.364 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.168.126:46569(127.7.168.126:46569)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:40.364 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.168.126:46569: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:40.365 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:40.365 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.168.125:46485(127.7.168.125:46485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:40.366 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.168.125:46485: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:40.368 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393; will retry
04:03:40.383 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.168.126:46569(127.7.168.126:46569)] connecting to peer
04:03:40.383 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.168.125:46485(127.7.168.125:46485)] connecting to peer
04:03:40.384 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:40.384 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.168.126:46569(127.7.168.126:46569)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:40.384 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.168.126:46569: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569
04:03:40.385 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:40.385 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.168.125:46485(127.7.168.125:46485)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:40.385 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.168.125:46485: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
04:03:40.387 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393; will retry
04:03:40.392 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=27147), Traces: [0ms] refreshing cache from master, [38ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.126:46569, [103ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.125:46485, [105ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.124:43393, [169ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.126:46569: Network error: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569, [167ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.125:46485: Network error: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [555ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.124:43393: OK, [559ms] delaying RPC due to: Service unavailable: Master config (127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [577ms] refreshing cache from master, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.126:46569, [580ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.125:46485, [581ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.124:43393, [588ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.126:46569: Network error: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569, [597ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.124:43393: OK, [598ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.125:46485: Network error: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [599ms] delaying RPC due to: Service unavailable: Master config (127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [617ms] refreshing cache from master, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.126:46569, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.125:46485, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.124:43393, [620ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.126:46569: Network error: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569, [624ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.125:46485: Network error: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [625ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.124:43393: OK, [626ms] delaying RPC due to: Service unavailable: Master config (127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [637ms] refreshing cache from master, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.126:46569, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.125:46485, [639ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.126:46569: Network error: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569, [639ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.124:43393, [641ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.125:46485: Network error: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [644ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.124:43393: OK, [646ms] delaying RPC due to: Service unavailable: Master config (127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [677ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.126:46569, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.125:46485, [680ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.126:46569: Network error: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569, [681ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.124:43393, [683ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.125:46485: Network error: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [685ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.124:43393: OK, [686ms] delaying RPC due to: Service unavailable: Master config (127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [717ms] refreshing cache from master, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.126:46569, [719ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.125:46485, [720ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.124:43393, [720ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.126:46569: Network error: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569, [724ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.125:46485: Network error: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [728ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.124:43393: OK, [729ms] delaying RPC due to: Service unavailable: Master config (127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [797ms] refreshing cache from master, [797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.126:46569, [799ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.125:46485, [800ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.124:43393, [802ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.126:46569: Network error: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569, [804ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.125:46485: Network error: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [806ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.124:43393: OK, [807ms] delaying RPC due to: Service unavailable: Master config (127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [936ms] refreshing cache from master, [937ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.126:46569, [938ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.125:46485, [940ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.124:43393, [942ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.126:46569: Network error: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569, [942ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.125:46485: Network error: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [946ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.124:43393: OK, [947ms] delaying RPC due to: Service unavailable: Master config (127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [1136ms] refreshing cache from master, [1137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.126:46569, [1138ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.125:46485, [1139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.124:43393, [1140ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.126:46569: Network error: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569, [1142ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.125:46485: Network error: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [1143ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.124:43393: OK, [1144ms] delaying RPC due to: Service unavailable: Master config (127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [1657ms] refreshing cache from master, [1657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.126:46569, [1658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.125:46485, [1660ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.126:46569: Network error: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569, [1661ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.124:43393, [1663ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.125:46485: Network error: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [1665ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.124:43393: OK, [1666ms] delaying RPC due to: Service unavailable: Master config (127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [1896ms] refreshing cache from master, [1897ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.126:46569, [1898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.125:46485, [1899ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.124:43393, [1900ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.126:46569: Network error: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569, [1901ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.125:46485: Network error: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [1903ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.124:43393: OK, [1903ms] delaying RPC due to: Service unavailable: Master config (127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [2677ms] refreshing cache from master, [2677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.126:46569, [2679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.125:46485, [2680ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.124:43393, [2681ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.126:46569: Network error: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569, [2682ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.125:46485: Network error: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [2684ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.124:43393: OK, [2684ms] delaying RPC due to: Service unavailable: Master config (127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [3237ms] refreshing cache from master, [3237ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.126:46569, [3238ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.125:46485, [3238ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.124:43393, [3239ms] too many traces: truncated at 100 traces, deferred=Deferred@483465(state=PENDING, result=null, callback=(continuation of Deferred@1528092905 after retry RPC after error@468391698) -> (continuation of Deferred@159603774 after retry RPC after error@851142319) -> (continuation of Deferred@1335178113 after retry RPC after error@210104038) -> (continuation of Deferred@527704548 after retry RPC after error@1989386939) -> (continuation of Deferred@276598483 after retry RPC after error@867016932) -> (continuation of Deferred@233364596 after retry RPC after error@1711612168) -> (continuation of Deferred@318451981 after retry RPC after error@1276053814) -> (continuation of Deferred@122644415 after retry RPC after error@1208818495) -> (continuation of Deferred@53815402 after retry RPC after error@935839619) -> (continuation of Deferred@1263086683 after retry RPC after error@1799160558) -> (continuation of Deferred@1096876443 after retry RPC after error@1217605099) -> (continuation of Deferred@1245379414 after retry RPC after error@1234418317) -> (continuation of Deferred@11995133 after retry RPC after error@1369083509) -> (continuation of Deferred@1281834843 after retry RPC after error@418689928) -> (continuation of Deferred@549771140 after retry RPC after error@623381711) -> (continuation of Deferred@1346396886 after retry RPC after error@1838543781) -> (continuation of Deferred@1802199396 after retry RPC after error@247486130) -> (continuation of Deferred@1573241710 after retry RPC after error@529468328) -> (continuation of Deferred@803158396 after retry RPC after error@589733875) -> (continuation of Deferred@1342961226 after retry RPC after error@926840232) -> (continuation of Deferred@1635565290 after retry RPC after error@1991404593) -> (continuation of Deferred@2038725899 after retry RPC after error@1037288614) -> (continuation of Deferred@970761752 after retry RPC after error@1558873846) -> (continuation of Deferred@106742853 after retry RPC after error@621401820) -> (continuation of Deferred@2042189827 after retry RPC after error@1902951668) -> (continuation of Deferred@292692080 after retry RPC after error@2057743284), errback=(continuation of Deferred@1528092905 after retry RPC after error@468391698) -> (continuation of Deferred@159603774 after retry RPC after error@851142319) -> (continuation of Deferred@1335178113 after retry RPC after error@210104038) -> (continuation of Deferred@527704548 after retry RPC after error@1989386939) -> (continuation of Deferred@276598483 after retry RPC after error@867016932) -> (continuation of Deferred@233364596 after retry RPC after error@1711612168) -> (continuation of Deferred@318451981 after retry RPC after error@1276053814) -> (continuation of Deferred@122644415 after retry RPC after error@1208818495) -> (continuation of Deferred@53815402 after retry RPC after error@935839619) -> (continuation of Deferred@1263086683 after retry RPC after error@1799160558) -> (continuation of Deferred@1096876443 after retry RPC after error@1217605099) -> (continuation of Deferred@1245379414 after retry RPC after error@1234418317) -> (continuation of Deferred@11995133 after retry RPC after error@1369083509) -> (continuation of Deferred@1281834843 after retry RPC after error@418689928) -> (continuation of Deferred@549771140 after retry RPC after error@623381711) -> (continuation of Deferred@1346396886 after retry RPC after error@1838543781) -> (continuation of Deferred@1802199396 after retry RPC after error@247486130) -> (continuation of Deferred@1573241710 after retry RPC after error@529468328) -> (continuation of Deferred@803158396 after retry RPC after error@589733875) -> (continuation of Deferred@1342961226 after retry RPC after error@926840232) -> (continuation of Deferred@1635565290 after retry RPC after error@1991404593) -> (continuation of Deferred@2038725899 after retry RPC after error@1037288614) -> (continuation of Deferred@970761752 after retry RPC after error@1558873846) -> (continuation of Deferred@106742853 after retry RPC after error@621401820) -> (continuation of Deferred@2042189827 after retry RPC after error@1902951668) -> (continuation of Deferred@292692080 after retry RPC after error@2057743284)))
04:03:40.396 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.7.168.124:43393(127.7.168.124:43393)] disconnecting while in state READY
04:03:40.397 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.7.168.125:46485(127.7.168.125:46485)] disconnecting while in state TERMINATED
04:03:40.398 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.7.168.126:46569(127.7.168.126:46569)] disconnecting while in state TERMINATED
04:03:40.402 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
04:03:40.409 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
04:03:40.411 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.7.168.124:43393(127.7.168.124:43393)] handling channelInactive
04:03:40.411 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.168.124:43393(127.7.168.124:43393)] cleaning up while in state READY due to: connection closed
04:03:40.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:40.420794 7841 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task4Qe23I/build/tsan/bin/kudu with pid 8276
04:03:41.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:41.260892 7686 debug-util.cc:398] Leaking SignalData structure 0x7b08000af9c0 after lost signal to thread 7561
04:03:41.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 04:03:41.262766 7686 debug-util.cc:398] Leaking SignalData structure 0x7b08000b9100 after lost signal to thread 7689
04:03:41.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-28T04:03:41Z chronyd exiting
04:03:42.738 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
04:03:42.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:42.740079 7113 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task4Qe23I/build/tsan/bin/kudu with pid 7392
04:03:42.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:42.767409 7113 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task4Qe23I/build/tsan/bin/kudu with pid 7553
04:03:42.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:42.795064 7113 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task4Qe23I/build/tsan/bin/kudu with pid 7700
04:03:42.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:42.820818 7113 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task4Qe23I/build/tsan/bin/kudu with pid 7125
04:03:42.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:42.851455 7113 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task4Qe23I/build/tsan/bin/kudu with pid 7195
04:03:42.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 04:03:42.880878 7113 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task4Qe23I/build/tsan/bin/kudu with pid 7270
04:03:44.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-28T04:03:44Z chronyd exiting
04:03:45.048 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=27147), Traces: [0ms] refreshing cache from master, [38ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.126:46569, [103ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.125:46485, [105ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.124:43393, [169ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.126:46569: Network error: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569, [167ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.125:46485: Network error: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [555ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.124:43393: OK, [559ms] delaying RPC due to: Service unavailable: Master config (127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [577ms] refreshing cache from master, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.126:46569, [580ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.125:46485, [581ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.124:43393, [588ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.126:46569: Network error: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569, [597ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.124:43393: OK, [598ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.125:46485: Network error: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [599ms] delaying RPC due to: Service unavailable: Master config (127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [617ms] refreshing cache from master, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.126:46569, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.125:46485, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.124:43393, [620ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.126:46569: Network error: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569, [624ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.125:46485: Network error: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [625ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.124:43393: OK, [626ms] delaying RPC due to: Service unavailable: Master config (127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [637ms] refreshing cache from master, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.126:46569, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.125:46485, [639ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.126:46569: Network error: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569, [639ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.124:43393, [641ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.125:46485: Network error: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [644ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.124:43393: OK, [646ms] delaying RPC due to: Service unavailable: Master config (127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [677ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.126:46569, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.125:46485, [680ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.126:46569: Network error: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569, [681ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.124:43393, [683ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.125:46485: Network error: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [685ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.124:43393: OK, [686ms] delaying RPC due to: Service unavailable: Master config (127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [717ms] refreshing cache from master, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.126:46569, [719ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.125:46485, [720ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.124:43393, [720ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.126:46569: Network error: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569, [724ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.125:46485: Network error: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [728ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.124:43393: OK, [729ms] delaying RPC due to: Service unavailable: Master config (127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [797ms] refreshing cache from master, [797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.126:46569, [799ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.125:46485, [800ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.124:43393, [802ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.126:46569: Network error: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569, [804ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.125:46485: Network error: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [806ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.124:43393: OK, [807ms] delaying RPC due to: Service unavailable: Master config (127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [936ms] refreshing cache from master, [937ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.126:46569, [938ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.125:46485, [940ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.124:43393, [942ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.126:46569: Network error: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569, [942ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.125:46485: Network error: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [946ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.124:43393: OK, [947ms] delaying RPC due to: Service unavailable: Master config (127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [1136ms] refreshing cache from master, [1137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.126:46569, [1138ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.125:46485, [1139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.124:43393, [1140ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.126:46569: Network error: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569, [1142ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.125:46485: Network error: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [1143ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.124:43393: OK, [1144ms] delaying RPC due to: Service unavailable: Master config (127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [1657ms] refreshing cache from master, [1657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.126:46569, [1658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.125:46485, [1660ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.126:46569: Network error: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569, [1661ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.124:43393, [1663ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.125:46485: Network error: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [1665ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.124:43393: OK, [1666ms] delaying RPC due to: Service unavailable: Master config (127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [1896ms] refreshing cache from master, [1897ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.126:46569, [1898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.125:46485, [1899ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.124:43393, [1900ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.126:46569: Network error: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569, [1901ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.125:46485: Network error: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [1903ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.124:43393: OK, [1903ms] delaying RPC due to: Service unavailable: Master config (127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [2677ms] refreshing cache from master, [2677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.126:46569, [2679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.125:46485, [2680ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.124:43393, [2681ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.126:46569: Network error: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569, [2682ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.125:46485: Network error: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [2684ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.124:43393: OK, [2684ms] delaying RPC due to: Service unavailable: Master config (127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [3237ms] refreshing cache from master, [3237ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.126:46569, [3238ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.125:46485, [3238ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.124:43393, [3239ms] too many traces: truncated at 100 traces, deferred=Deferred@483465(state=PENDING, result=null, callback=(continuation of Deferred@1528092905 after retry RPC after error@468391698) -> (continuation of Deferred@159603774 after retry RPC after error@851142319) -> (continuation of Deferred@1335178113 after retry RPC after error@210104038) -> (continuation of Deferred@527704548 after retry RPC after error@1989386939) -> (continuation of Deferred@276598483 after retry RPC after error@867016932) -> (continuation of Deferred@233364596 after retry RPC after error@1711612168) -> (continuation of Deferred@318451981 after retry RPC after error@1276053814) -> (continuation of Deferred@122644415 after retry RPC after error@1208818495) -> (continuation of Deferred@53815402 after retry RPC after error@935839619) -> (continuation of Deferred@1263086683 after retry RPC after error@1799160558) -> (continuation of Deferred@1096876443 after retry RPC after error@1217605099) -> (continuation of Deferred@1245379414 after retry RPC after error@1234418317) -> (continuation of Deferred@11995133 after retry RPC after error@1369083509) -> (continuation of Deferred@1281834843 after retry RPC after error@418689928) -> (continuation of Deferred@549771140 after retry RPC after error@623381711) -> (continuation of Deferred@1346396886 after retry RPC after error@1838543781) -> (continuation of Deferred@1802199396 after retry RPC after error@247486130) -> (continuation of Deferred@1573241710 after retry RPC after error@529468328) -> (continuation of Deferred@803158396 after retry RPC after error@589733875) -> (continuation of Deferred@1342961226 after retry RPC after error@926840232) -> (continuation of Deferred@1635565290 after retry RPC after error@1991404593) -> (continuation of Deferred@2038725899 after retry RPC after error@1037288614) -> (continuation of Deferred@970761752 after retry RPC after error@1558873846) -> (continuation of Deferred@106742853 after retry RPC after error@621401820) -> (continuation of Deferred@2042189827 after retry RPC after error@1902951668) -> (continuation of Deferred@292692080 after retry RPC after error@2057743284), errback=(continuation of Deferred@1528092905 after retry RPC after error@468391698) -> (continuation of Deferred@159603774 after retry RPC after error@851142319) -> (continuation of Deferred@1335178113 after retry RPC after error@210104038) -> (continuation of Deferred@527704548 after retry RPC after error@1989386939) -> (continuation of Deferred@276598483 after retry RPC after error@867016932) -> (continuation of Deferred@233364596 after retry RPC after error@1711612168) -> (continuation of Deferred@318451981 after retry RPC after error@1276053814) -> (continuation of Deferred@122644415 after retry RPC after error@1208818495) -> (continuation of Deferred@53815402 after retry RPC after error@935839619) -> (continuation of Deferred@1263086683 after retry RPC after error@1799160558) -> (continuation of Deferred@1096876443 after retry RPC after error@1217605099) -> (continuation of Deferred@1245379414 after retry RPC after error@1234418317) -> (continuation of Deferred@11995133 after retry RPC after error@1369083509) -> (continuation of Deferred@1281834843 after retry RPC after error@418689928) -> (continuation of Deferred@549771140 after retry RPC after error@623381711) -> (continuation of Deferred@1346396886 after retry RPC after error@1838543781) -> (continuation of Deferred@1802199396 after retry RPC after error@247486130) -> (continuation of Deferred@1573241710 after retry RPC after error@529468328) -> (continuation of Deferred@803158396 after retry RPC after error@589733875) -> (continuation of Deferred@1342961226 after retry RPC after error@926840232) -> (continuation of Deferred@1635565290 after retry RPC after error@1991404593) -> (continuation of Deferred@2038725899 after retry RPC after error@1037288614) -> (continuation of Deferred@970761752 after retry RPC after error@1558873846) -> (continuation of Deferred@106742853 after retry RPC after error@621401820) -> (continuation of Deferred@2042189827 after retry RPC after error@1902951668) -> (continuation of Deferred@292692080 after retry RPC after error@2057743284)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.7.168.125:46485
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=27147), Traces: [0ms] refreshing cache from master, [38ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.126:46569, [103ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.125:46485, [105ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.124:43393, [169ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.126:46569: Network error: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569, [167ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.125:46485: Network error: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [555ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.124:43393: OK, [559ms] delaying RPC due to: Service unavailable: Master config (127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [577ms] refreshing cache from master, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.126:46569, [580ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.125:46485, [581ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.124:43393, [588ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.126:46569: Network error: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569, [597ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.124:43393: OK, [598ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.125:46485: Network error: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [599ms] delaying RPC due to: Service unavailable: Master config (127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [617ms] refreshing cache from master, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.126:46569, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.125:46485, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.124:43393, [620ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.126:46569: Network error: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569, [624ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.125:46485: Network error: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [625ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.124:43393: OK, [626ms] delaying RPC due to: Service unavailable: Master config (127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [637ms] refreshing cache from master, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.126:46569, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.125:46485, [639ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.126:46569: Network error: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569, [639ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.124:43393, [641ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.125:46485: Network error: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [644ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.124:43393: OK, [646ms] delaying RPC due to: Service unavailable: Master config (127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [677ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.126:46569, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.125:46485, [680ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.126:46569: Network error: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569, [681ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.124:43393, [683ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.125:46485: Network error: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [685ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.124:43393: OK, [686ms] delaying RPC due to: Service unavailable: Master config (127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [717ms] refreshing cache from master, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.126:46569, [719ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.125:46485, [720ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.124:43393, [720ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.126:46569: Network error: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569, [724ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.125:46485: Network error: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [728ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.124:43393: OK, [729ms] delaying RPC due to: Service unavailable: Master config (127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [797ms] refreshing cache from master, [797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.126:46569, [799ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.125:46485, [800ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.124:43393, [802ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.126:46569: Network error: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569, [804ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.125:46485: Network error: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [806ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.124:43393: OK, [807ms] delaying RPC due to: Service unavailable: Master config (127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [936ms] refreshing cache from master, [937ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.126:46569, [938ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.125:46485, [940ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.124:43393, [942ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.126:46569: Network error: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569, [942ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.125:46485: Network error: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [946ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.124:43393: OK, [947ms] delaying RPC due to: Service unavailable: Master config (127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [1136ms] refreshing cache from master, [1137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.126:46569, [1138ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.125:46485, [1139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.124:43393, [1140ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.126:46569: Network error: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569, [1142ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.125:46485: Network error: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [1143ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.124:43393: OK, [1144ms] delaying RPC due to: Service unavailable: Master config (127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [1657ms] refreshing cache from master, [1657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.126:46569, [1658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.125:46485, [1660ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.126:46569: Network error: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569, [1661ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.124:43393, [1663ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.125:46485: Network error: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [1665ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.124:43393: OK, [1666ms] delaying RPC due to: Service unavailable: Master config (127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [1896ms] refreshing cache from master, [1897ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.126:46569, [1898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.125:46485, [1899ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.124:43393, [1900ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.126:46569: Network error: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569, [1901ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.125:46485: Network error: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [1903ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.124:43393: OK, [1903ms] delaying RPC due to: Service unavailable: Master config (127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [2677ms] refreshing cache from master, [2677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.126:46569, [2679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.125:46485, [2680ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.124:43393, [2681ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.126:46569: Network error: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569, [2682ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.125:46485: Network error: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [2684ms] Sub RPC ConnectToMaster: received response from server master-127.7.168.124:43393: OK, [2684ms] delaying RPC due to: Service unavailable: Master config (127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485, [3237ms] refreshing cache from master, [3237ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.126:46569, [3238ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.125:46485, [3238ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.168.124:43393, [3239ms] too many traces: truncated at 100 traces, deferred=Deferred@483465(state=PENDING, result=null, callback=(continuation of Deferred@1528092905 after retry RPC after error@468391698) -> (continuation of Deferred@159603774 after retry RPC after error@851142319) -> (continuation of Deferred@1335178113 after retry RPC after error@210104038) -> (continuation of Deferred@527704548 after retry RPC after error@1989386939) -> (continuation of Deferred@276598483 after retry RPC after error@867016932) -> (continuation of Deferred@233364596 after retry RPC after error@1711612168) -> (continuation of Deferred@318451981 after retry RPC after error@1276053814) -> (continuation of Deferred@122644415 after retry RPC after error@1208818495) -> (continuation of Deferred@53815402 after retry RPC after error@935839619) -> (continuation of Deferred@1263086683 after retry RPC after error@1799160558) -> (continuation of Deferred@1096876443 after retry RPC after error@1217605099) -> (continuation of Deferred@1245379414 after retry RPC after error@1234418317) -> (continuation of Deferred@11995133 after retry RPC after error@1369083509) -> (continuation of Deferred@1281834843 after retry RPC after error@418689928) -> (continuation of Deferred@549771140 after retry RPC after error@623381711) -> (continuation of Deferred@1346396886 after retry RPC after error@1838543781) -> (continuation of Deferred@1802199396 after retry RPC after error@247486130) -> (continuation of Deferred@1573241710 after retry RPC after error@529468328) -> (continuation of Deferred@803158396 after retry RPC after error@589733875) -> (continuation of Deferred@1342961226 after retry RPC after error@926840232) -> (continuation of Deferred@1635565290 after retry RPC after error@1991404593) -> (continuation of Deferred@2038725899 after retry RPC after error@1037288614) -> (continuation of Deferred@970761752 after retry RPC after error@1558873846) -> (continuation of Deferred@106742853 after retry RPC after error@621401820) -> (continuation of Deferred@2042189827 after retry RPC after error@1902951668) -> (continuation of Deferred@292692080 after retry RPC after error@2057743284), errback=(continuation of Deferred@1528092905 after retry RPC after error@468391698) -> (continuation of Deferred@159603774 after retry RPC after error@851142319) -> (continuation of Deferred@1335178113 after retry RPC after error@210104038) -> (continuation of Deferred@527704548 after retry RPC after error@1989386939) -> (continuation of Deferred@276598483 after retry RPC after error@867016932) -> (continuation of Deferred@233364596 after retry RPC after error@1711612168) -> (continuation of Deferred@318451981 after retry RPC after error@1276053814) -> (continuation of Deferred@122644415 after retry RPC after error@1208818495) -> (continuation of Deferred@53815402 after retry RPC after error@935839619) -> (continuation of Deferred@1263086683 after retry RPC after error@1799160558) -> (continuation of Deferred@1096876443 after retry RPC after error@1217605099) -> (continuation of Deferred@1245379414 after retry RPC after error@1234418317) -> (continuation of Deferred@11995133 after retry RPC after error@1369083509) -> (continuation of Deferred@1281834843 after retry RPC after error@418689928) -> (continuation of Deferred@549771140 after retry RPC after error@623381711) -> (continuation of Deferred@1346396886 after retry RPC after error@1838543781) -> (continuation of Deferred@1802199396 after retry RPC after error@247486130) -> (continuation of Deferred@1573241710 after retry RPC after error@529468328) -> (continuation of Deferred@803158396 after retry RPC after error@589733875) -> (continuation of Deferred@1342961226 after retry RPC after error@926840232) -> (continuation of Deferred@1635565290 after retry RPC after error@1991404593) -> (continuation of Deferred@2038725899 after retry RPC after error@1037288614) -> (continuation of Deferred@970761752 after retry RPC after error@1558873846) -> (continuation of Deferred@106742853 after retry RPC after error@621401820) -> (continuation of Deferred@2042189827 after retry RPC after error@1902951668) -> (continuation of Deferred@292692080 after retry RPC after error@2057743284)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.7.168.126:46569,127.7.168.125:46485,127.7.168.124:43393) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.126:46569(127.7.168.126:46569): Connection refused: /127.7.168.126:46569,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.168.125:46485(127.7.168.125:46485): Connection refused: /127.7.168.125:46485
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.7.168.125:46485
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)