16:30:28.165 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
16:30:28.176 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task4YrI62/test-tmp
16:30:28.179 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
16:30:28.182 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task4YrI62/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
16:30:28.411 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
16:30:28.609 [DEBUG - main] (MiniKuduCluster.java:195) Response:
16:30:28.610 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
16:30:28.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:28.615235 4615 mini_oidc.cc:152] Starting JWKS server
16:30:28.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:28.619731 4615 webserver.cc:460] Webserver started at http://127.0.0.1:34981/ using document root <none> and password file <none>
16:30:28.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:28.620217 4615 mini_oidc.cc:189] Starting OIDC Discovery server
16:30:28.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:28.621914 4615 webserver.cc:460] Webserver started at http://127.0.0.1:44841/ using document root <none> and password file <none>
16:30:28.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-26T16:30:28Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
16:30:28.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-26T16:30:28Z Disabled control of system clock
16:30:28.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:28.676470 4615 external_mini_cluster.cc:1326] Running /tmp/dist-test-task4YrI62/build/tsan/bin/kudu
16:30:28.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4YrI62/build/tsan/bin/kudu
16:30:28.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/master-0/wal
16:30:28.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/master-0/data
16:30:28.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:30:28.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:30:28.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:30:28.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:30:28.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:30:28.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:30:28.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/master-0/logs
16:30:28.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/master-0/data/info.pb
16:30:28.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:30:28.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:30:28.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:30:28.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:30:28.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:30:28.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:30:28.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:30:28.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:30:28.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:30:28.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:30:28.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:30:28.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:30:28.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.129.254:36341
16:30:28.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.129.254
16:30:28.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:30:28.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.129.212:43985
16:30:28.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:30:28.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:30:28.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:30:28.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:30:28.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.129.254:36341,127.4.129.253:39681,127.4.129.252:39793
16:30:28.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:30:28.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34981/jwks.json with env {}
16:30:28.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:28.985833 4627 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:30:28.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:28.986480 4627 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:30:28.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:28.987246 4627 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:30:29.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:29.018539 4627 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:30:29.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:29.018831 4627 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:30:29.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:29.019026 4627 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:30:29.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:29.019284 4627 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:30:29.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:29.019565 4627 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:30:29.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:29.019786 4627 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34981/jwks.json
16:30:29.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:29.056020 4627 master_runner.cc:386] Master server non-default flags:
16:30:29.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:30:29.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.129.212:43985
16:30:29.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:30:29.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:30:29.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/master-0/data
16:30:29.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/master-0/wal
16:30:29.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:30:29.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.129.254:36341,127.4.129.253:39681,127.4.129.252:39793
16:30:29.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:30:29.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:30:29.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:30:29.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.129.254:36341
16:30:29.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:30:29.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:30:29.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:30:29.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34981/jwks.json
16:30:29.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:30:29.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:30:29.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/master-0/data/info.pb
16:30:29.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.129.254
16:30:29.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:30:29.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:30:29.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:30:29.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:30:29.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:30:29.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:30:29.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/master-0/logs
16:30:29.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:30:29.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:30:29.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:30:29.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:30:29.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
16:30:29.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
16:30:29.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:30:29.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 15:11:41 UTC on ccycloud.quasar-yukscv.root.comops.site
16:30:29.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4674342
16:30:29.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:30:29.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:29.057418 4627 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:30:29.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:29.059597 4627 file_cache.cc:493] Constructed file cache file cache with capacity 419430
16:30:29.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:29.078758 4636 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:30:30.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:29.078819 4634 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:30:30.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:29.079981 4627 server_base.cc:1034] running on GCE node
16:30:30.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:29.078995 4633 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:30:30.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:30.292034 4627 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:30:30.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:30.294961 4627 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:30:30.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:30.296420 4627 hybrid_clock.cc:648] HybridClock initialized: now 1790440230296366 us; error 64 us; skew 500 ppm
16:30:30.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:30.297286 4627 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:30:30.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:30.312194 4627 webserver.cc:460] Webserver started at http://127.4.129.254:45147/ using document root <none> and password file <none>
16:30:30.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:30.313365 4627 fs_manager.cc:359] Metadata directory not provided
16:30:30.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:30.313673 4627 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:30:30.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:30.314303 4627 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
16:30:30.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:30.325968 4627 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/master-0/data/instance:
16:30:30.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cc68485f0c2844df88f7839fa23f03f0"
16:30:30.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:30:30 on dist-test-slave-lbns"
16:30:30.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:30.327430 4627 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/master-0/wal/instance:
16:30:30.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cc68485f0c2844df88f7839fa23f03f0"
16:30:30.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:30:30 on dist-test-slave-lbns"
16:30:30.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:30.336184 4627 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.000s sys 0.008s
16:30:30.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:30.342083 4643 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:30:30.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:30.343418 4627 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.000s
16:30:30.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:30.343796 4627 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/master-0/data,/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/master-0/wal
16:30:30.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cc68485f0c2844df88f7839fa23f03f0"
16:30:30.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:30:30 on dist-test-slave-lbns"
16:30:30.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:30.344133 4627 fs_report.cc:383] FS layout report
16:30:30.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:30:30.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/master-0/wal
16:30:30.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/master-0/wal
16:30:30.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/master-0/data/data
16:30:30.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:30:30.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:30:30.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:30:30.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:30:30.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:30:30.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:30:30.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:30:30.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:30:30.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:30:30.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:30:30.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:30:30.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:30.390128 4627 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
16:30:30.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:30.392584 4627 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:30:30.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:30.393314 4627 kserver.cc:163] Server-wide thread pool size limit: 3276
16:30:30.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:30.471197 4627 rpc_server.cc:307] RPC server started. Bound to: 127.4.129.254:36341
16:30:30.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:30.471289 4694 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.129.254:36341 every 8 connection(s)
16:30:30.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:30.473847 4627 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/master-0/data/info.pb
16:30:30.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:30.475788 4615 external_mini_cluster.cc:1388] Started /tmp/dist-test-task4YrI62/build/tsan/bin/kudu as pid 4627
16:30:30.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:30.476315 4615 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/master-0/wal/instance
16:30:30.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:30.480216 4695 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
16:30:30.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:30.480556 4615 external_mini_cluster.cc:1326] Running /tmp/dist-test-task4YrI62/build/tsan/bin/kudu
16:30:30.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4YrI62/build/tsan/bin/kudu
16:30:30.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/master-1/wal
16:30:30.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/master-1/data
16:30:30.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:30:30.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:30:30.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:30:30.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:30:30.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:30:30.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:30:30.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/master-1/logs
16:30:30.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/master-1/data/info.pb
16:30:30.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:30:30.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:30:30.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:30:30.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:30:30.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:30:30.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:30:30.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:30:30.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:30:30.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:30:30.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:30:30.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:30:30.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:30:30.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.129.253:39681
16:30:30.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.129.253
16:30:30.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:30:30.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.129.212:43985
16:30:30.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:30:30.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:30:30.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:30:30.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:30:30.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.129.254:36341,127.4.129.253:39681,127.4.129.252:39793
16:30:30.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:30:30.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34981/jwks.json with env {}
16:30:30.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:30.494288 4695 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.129.254" port: 36341 } has no permanent_uuid. Determining permanent_uuid...
16:30:30.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:30.525686 4695 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.129.253" port: 39681 } has no permanent_uuid. Determining permanent_uuid...
16:30:30.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:30.527998 4644 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.129.253:39681: connect: Connection refused (error 111)
16:30:30.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:30.532042 4695 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.129.253:39681: Network error: Client connection negotiation failed: client connection to 127.4.129.253:39681: connect: Connection refused (error 111)
16:30:30.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:30.581610 4695 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.129.253" port: 39681 } attempt: 1
16:30:30.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:30.585548 4695 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.129.253:39681: Network error: Client connection negotiation failed: client connection to 127.4.129.253:39681: connect: Connection refused (error 111)
16:30:30.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:30.654043 4695 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.129.253" port: 39681 } attempt: 2
16:30:30.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:30.658597 4695 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.129.253:39681: Network error: Client connection negotiation failed: client connection to 127.4.129.253:39681: connect: Connection refused (error 111)
16:30:30.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:30.750093 4695 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.129.253" port: 39681 } attempt: 3
16:30:30.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:30.754151 4695 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.129.253:39681: Network error: Client connection negotiation failed: client connection to 127.4.129.253:39681: connect: Connection refused (error 111)
16:30:30.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:30.807298 4697 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:30:30.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:30.807874 4697 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:30:30.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:30.808447 4697 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:30:30.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:30.839447 4697 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:30:30.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:30.839747 4697 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:30:30.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:30.839936 4697 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:30:30.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:30.840237 4697 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:30:30.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:30.840569 4697 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:30:30.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:30.840763 4697 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34981/jwks.json
16:30:30.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:30.879274 4697 master_runner.cc:386] Master server non-default flags:
16:30:30.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:30:30.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.129.212:43985
16:30:30.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:30:30.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:30:30.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/master-1/data
16:30:30.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/master-1/wal
16:30:30.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:30:30.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.129.254:36341,127.4.129.253:39681,127.4.129.252:39793
16:30:30.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:30:30.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:30:30.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:30:30.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.129.253:39681
16:30:30.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:30:30.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:30:30.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:30:30.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34981/jwks.json
16:30:30.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:30:30.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:30:30.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/master-1/data/info.pb
16:30:30.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.129.253
16:30:30.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:30:30.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:30:30.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:30:30.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:30:30.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:30:30.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:30:30.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/master-1/logs
16:30:30.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:30:30.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:30:30.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:30:30.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:30:30.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
16:30:30.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
16:30:30.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:30:30.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 15:11:41 UTC on ccycloud.quasar-yukscv.root.comops.site
16:30:30.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4674342
16:30:30.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:30:30.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:30.880477 4697 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:30:30.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:30.882223 4697 file_cache.cc:493] Constructed file cache file cache with capacity 419430
16:30:30.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:30.897589 4695 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.129.253" port: 39681 } attempt: 4
16:30:30.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:30.900152 4709 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:30:30.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:30.903133 4695 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.129.253:39681: Network error: Client connection negotiation failed: client connection to 127.4.129.253:39681: connect: Connection refused (error 111)
16:30:31.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:31.202597 4695 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.129.253" port: 39681 } attempt: 5
16:30:31.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:31.206642 4695 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.129.253:39681: Network error: Client connection negotiation failed: client connection to 127.4.129.253:39681: connect: Connection refused (error 111)
16:30:31.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:31.754259 4695 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.129.253" port: 39681 } attempt: 6
16:30:31.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:31.761379 4695 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.129.253:39681: Network error: Client connection negotiation failed: client connection to 127.4.129.253:39681: connect: Connection refused (error 111)
16:30:32.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:30.901496 4707 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:30:32.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:30.902846 4706 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:30:32.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:30.901973 4697 server_base.cc:1034] running on GCE node
16:30:32.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:32.152818 4697 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:30:32.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:32.155810 4697 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:30:32.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:32.157208 4697 hybrid_clock.cc:648] HybridClock initialized: now 1790440232157192 us; error 70 us; skew 500 ppm
16:30:32.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:32.158209 4697 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:30:32.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:32.164944 4697 webserver.cc:460] Webserver started at http://127.4.129.253:35981/ using document root <none> and password file <none>
16:30:32.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:32.166049 4697 fs_manager.cc:359] Metadata directory not provided
16:30:32.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:32.166313 4697 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:30:32.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:32.166846 4697 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
16:30:32.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:32.171819 4697 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/master-1/data/instance:
16:30:32.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0920946e0a6b490a83d891822679cd0c"
16:30:32.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:30:32 on dist-test-slave-lbns"
16:30:32.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:32.173077 4697 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/master-1/wal/instance:
16:30:32.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0920946e0a6b490a83d891822679cd0c"
16:30:32.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:30:32 on dist-test-slave-lbns"
16:30:32.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:32.180393 4697 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.004s
16:30:32.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:32.186386 4718 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:30:32.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:32.191236 4697 fs_manager.cc:725] Time spent opening block manager: real 0.008s user 0.001s sys 0.002s
16:30:32.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:32.191728 4697 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/master-1/data,/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/master-1/wal
16:30:32.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0920946e0a6b490a83d891822679cd0c"
16:30:32.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:30:32 on dist-test-slave-lbns"
16:30:32.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:32.192085 4697 fs_report.cc:383] FS layout report
16:30:32.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:30:32.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/master-1/wal
16:30:32.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/master-1/wal
16:30:32.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/master-1/data/data
16:30:32.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:30:32.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:30:32.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:30:32.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:30:32.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:30:32.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:30:32.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:30:32.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:30:32.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:30:32.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:30:32.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:30:32.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:32.246069 4697 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
16:30:32.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:32.247710 4697 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:30:32.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:32.248139 4697 kserver.cc:163] Server-wide thread pool size limit: 3276
16:30:32.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:32.320485 4697 rpc_server.cc:307] RPC server started. Bound to: 127.4.129.253:39681
16:30:32.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:32.320634 4769 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.129.253:39681 every 8 connection(s)
16:30:32.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:32.323184 4697 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/master-1/data/info.pb
16:30:32.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:32.328155 4770 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
16:30:32.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:32.331171 4615 external_mini_cluster.cc:1388] Started /tmp/dist-test-task4YrI62/build/tsan/bin/kudu as pid 4697
16:30:32.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:32.331641 4615 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/master-1/wal/instance
16:30:32.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:32.334599 4615 external_mini_cluster.cc:1326] Running /tmp/dist-test-task4YrI62/build/tsan/bin/kudu
16:30:32.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4YrI62/build/tsan/bin/kudu
16:30:32.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/master-2/wal
16:30:32.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/master-2/data
16:30:32.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:30:32.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:30:32.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:30:32.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:30:32.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:30:32.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:30:32.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/master-2/logs
16:30:32.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/master-2/data/info.pb
16:30:32.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:30:32.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:30:32.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:30:32.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:30:32.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:30:32.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:30:32.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:30:32.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:30:32.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:30:32.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:30:32.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:30:32.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:30:32.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.129.252:39793
16:30:32.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.129.252
16:30:32.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:30:32.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.129.212:43985
16:30:32.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:30:32.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:30:32.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:30:32.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:30:32.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.129.254:36341,127.4.129.253:39681,127.4.129.252:39793
16:30:32.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:30:32.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34981/jwks.json with env {}
16:30:32.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:32.343997 4770 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.129.254" port: 36341 } has no permanent_uuid. Determining permanent_uuid...
16:30:32.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:32.370306 4770 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.129.253" port: 39681 } has no permanent_uuid. Determining permanent_uuid...
16:30:32.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:32.387998 4770 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.129.252" port: 39793 } has no permanent_uuid. Determining permanent_uuid...
16:30:32.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:32.390479 4719 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.129.252:39793: connect: Connection refused (error 111)
16:30:32.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:32.396792 4770 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.129.252:39793: Network error: Client connection negotiation failed: client connection to 127.4.129.252:39793: connect: Connection refused (error 111)
16:30:32.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:32.446383 4770 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.129.252" port: 39793 } attempt: 1
16:30:32.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:32.450280 4770 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.129.252:39793: Network error: Client connection negotiation failed: client connection to 127.4.129.252:39793: connect: Connection refused (error 111)
16:30:32.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:32.518721 4770 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.129.252" port: 39793 } attempt: 2
16:30:32.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:32.522641 4770 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.129.252:39793: Network error: Client connection negotiation failed: client connection to 127.4.129.252:39793: connect: Connection refused (error 111)
16:30:32.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:32.614045 4770 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.129.252" port: 39793 } attempt: 3
16:30:32.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:32.618110 4770 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.129.252:39793: Network error: Client connection negotiation failed: client connection to 127.4.129.252:39793: connect: Connection refused (error 111)
16:30:32.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:32.647819 4772 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:30:32.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:32.648434 4772 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:30:32.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:32.648945 4772 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:30:32.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:32.680096 4772 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:30:32.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:32.680378 4772 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:30:32.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:32.680562 4772 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:30:32.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:32.680737 4772 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:30:32.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:32.680919 4772 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:30:32.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:32.681123 4772 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34981/jwks.json
16:30:32.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:32.716928 4772 master_runner.cc:386] Master server non-default flags:
16:30:32.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:30:32.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.129.212:43985
16:30:32.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:30:32.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:30:32.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/master-2/data
16:30:32.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/master-2/wal
16:30:32.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:30:32.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.129.254:36341,127.4.129.253:39681,127.4.129.252:39793
16:30:32.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:30:32.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:30:32.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:30:32.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.129.252:39793
16:30:32.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:30:32.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:30:32.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:30:32.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34981/jwks.json
16:30:32.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:30:32.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:30:32.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/master-2/data/info.pb
16:30:32.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.129.252
16:30:32.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:30:32.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:30:32.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:30:32.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:30:32.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:30:32.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:30:32.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/master-2/logs
16:30:32.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:30:32.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:30:32.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:30:32.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:30:32.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
16:30:32.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
16:30:32.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:30:32.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 15:11:41 UTC on ccycloud.quasar-yukscv.root.comops.site
16:30:32.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4674342
16:30:32.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:30:32.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:32.718084 4772 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:30:32.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:32.720079 4772 file_cache.cc:493] Constructed file cache file cache with capacity 419430
16:30:32.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:32.738873 4783 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:30:32.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:32.761569 4770 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.129.252" port: 39793 } attempt: 4
16:30:32.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:32.766377 4770 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.129.252:39793: Network error: Client connection negotiation failed: client connection to 127.4.129.252:39793: connect: Connection refused (error 111)
16:30:32.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:32.827931 4695 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.129.253" port: 39681 } attempt: 7
16:30:32.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:32.845175 4695 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.129.252" port: 39793 } has no permanent_uuid. Determining permanent_uuid...
16:30:32.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:32.851944 4695 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.129.252:39793: Network error: Client connection negotiation failed: client connection to 127.4.129.252:39793: connect: Connection refused (error 111)
16:30:32.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:32.910450 4695 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.129.252" port: 39793 } attempt: 1
16:30:32.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:32.914563 4695 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.129.252:39793: Network error: Client connection negotiation failed: client connection to 127.4.129.252:39793: connect: Connection refused (error 111)
16:30:32.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:32.996222 4695 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.129.252" port: 39793 } attempt: 2
16:30:33.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:33.001339 4695 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.129.252:39793: Network error: Client connection negotiation failed: client connection to 127.4.129.252:39793: connect: Connection refused (error 111)
16:30:33.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:33.065860 4770 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.129.252" port: 39793 } attempt: 5
16:30:33.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:33.070781 4770 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.129.252:39793: Network error: Client connection negotiation failed: client connection to 127.4.129.252:39793: connect: Connection refused (error 111)
16:30:33.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:33.086737 4695 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.129.252" port: 39793 } attempt: 3
16:30:33.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:33.091228 4695 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.129.252:39793: Network error: Client connection negotiation failed: client connection to 127.4.129.252:39793: connect: Connection refused (error 111)
16:30:33.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:33.227555 4691 debug-util.cc:398] Leaking SignalData structure 0x7b0800072760 after lost signal to thread 4628
16:30:33.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:33.231679 4695 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.129.252" port: 39793 } attempt: 4
16:30:33.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:33.237603 4695 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.129.252:39793: Network error: Client connection negotiation failed: client connection to 127.4.129.252:39793: connect: Connection refused (error 111)
16:30:33.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:33.521153 4695 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.129.252" port: 39793 } attempt: 5
16:30:33.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:33.526243 4695 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.129.252:39793: Network error: Client connection negotiation failed: client connection to 127.4.129.252:39793: connect: Connection refused (error 111)
16:30:33.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:33.618413 4770 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.129.252" port: 39793 } attempt: 6
16:30:33.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:33.629745 4770 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.129.252:39793: Network error: Client connection negotiation failed: client connection to 127.4.129.252:39793: connect: Connection refused (error 111)
16:30:33.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:32.739173 4782 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:30:33.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:32.741174 4785 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:30:33.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:33.971637 4784 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
16:30:33.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:33.972093 4772 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:30:33.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:33.976544 4772 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:30:33.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:33.980003 4772 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:30:33.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:33.981529 4772 hybrid_clock.cc:648] HybridClock initialized: now 1790440233981502 us; error 48 us; skew 500 ppm
16:30:33.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:33.982430 4772 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:30:33.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:33.990108 4772 webserver.cc:460] Webserver started at http://127.4.129.252:37457/ using document root <none> and password file <none>
16:30:33.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:33.991158 4772 fs_manager.cc:359] Metadata directory not provided
16:30:33.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:33.991413 4772 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:30:33.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:33.991973 4772 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
16:30:34.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:33.999315 4772 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/master-2/data/instance:
16:30:34.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fe8426faeb6a4fc898c723f1e66f1298"
16:30:34.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:30:33 on dist-test-slave-lbns"
16:30:34.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.000939 4772 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/master-2/wal/instance:
16:30:34.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fe8426faeb6a4fc898c723f1e66f1298"
16:30:34.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:30:33 on dist-test-slave-lbns"
16:30:34.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.011030 4772 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.010s sys 0.001s
16:30:34.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.018411 4796 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:30:34.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.020098 4772 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.004s sys 0.000s
16:30:34.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.020560 4772 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/master-2/data,/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/master-2/wal
16:30:34.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fe8426faeb6a4fc898c723f1e66f1298"
16:30:34.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:30:33 on dist-test-slave-lbns"
16:30:34.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.021059 4772 fs_report.cc:383] FS layout report
16:30:34.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:30:34.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/master-2/wal
16:30:34.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/master-2/wal
16:30:34.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/master-2/data/data
16:30:34.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:30:34.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:30:34.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:30:34.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:30:34.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:30:34.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:30:34.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:30:34.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:30:34.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:30:34.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:30:34.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:30:34.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.078756 4695 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.129.252" port: 39793 } attempt: 6
16:30:34.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:34.086695 4695 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.129.252:39793: Network error: Client connection negotiation failed: client connection to 127.4.129.252:39793: connect: Connection refused (error 111)
16:30:34.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.093645 4772 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
16:30:34.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.095912 4772 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:30:34.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.096570 4772 kserver.cc:163] Server-wide thread pool size limit: 3276
16:30:34.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.191983 4772 rpc_server.cc:307] RPC server started. Bound to: 127.4.129.252:39793
16:30:34.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.192016 4849 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.129.252:39793 every 8 connection(s)
16:30:34.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.196614 4772 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/master-2/data/info.pb
16:30:34.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.203172 4850 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
16:30:34.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.203254 4615 external_mini_cluster.cc:1388] Started /tmp/dist-test-task4YrI62/build/tsan/bin/kudu as pid 4772
16:30:34.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.203701 4615 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/master-2/wal/instance
16:30:34.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.223646 4850 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.129.254" port: 36341 } has no permanent_uuid. Determining permanent_uuid...
16:30:34.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.265697 4850 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.129.253" port: 39681 } has no permanent_uuid. Determining permanent_uuid...
16:30:34.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.279850 4850 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.129.252" port: 39793 } has no permanent_uuid. Determining permanent_uuid...
16:30:34.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.300534 4850 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P fe8426faeb6a4fc898c723f1e66f1298: Bootstrap starting.
16:30:34.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.308225 4850 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P fe8426faeb6a4fc898c723f1e66f1298: Neither blocks nor log segments found. Creating new log.
16:30:34.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.311173 4850 log.cc:824] T 00000000000000000000000000000000 P fe8426faeb6a4fc898c723f1e66f1298: Log is configured to *not* fsync() on all Append() calls
16:30:34.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.321069 4850 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P fe8426faeb6a4fc898c723f1e66f1298: No bootstrap required, opened a new log
16:30:34.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.355132 4850 raft_consensus.cc:348] T 00000000000000000000000000000000 P fe8426faeb6a4fc898c723f1e66f1298 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cc68485f0c2844df88f7839fa23f03f0" member_type: VOTER last_known_addr { host: "127.4.129.254" port: 36341 } } peers { permanent_uuid: "0920946e0a6b490a83d891822679cd0c" member_type: VOTER last_known_addr { host: "127.4.129.253" port: 39681 } } peers { permanent_uuid: "fe8426faeb6a4fc898c723f1e66f1298" member_type: VOTER last_known_addr { host: "127.4.129.252" port: 39793 } }
16:30:34.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.356527 4850 raft_consensus.cc:374] T 00000000000000000000000000000000 P fe8426faeb6a4fc898c723f1e66f1298 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
16:30:34.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.356900 4850 raft_consensus.cc:729] T 00000000000000000000000000000000 P fe8426faeb6a4fc898c723f1e66f1298 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fe8426faeb6a4fc898c723f1e66f1298, State: Initialized, Role: FOLLOWER
16:30:34.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.358004 4850 consensus_queue.cc:260] T 00000000000000000000000000000000 P fe8426faeb6a4fc898c723f1e66f1298 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cc68485f0c2844df88f7839fa23f03f0" member_type: VOTER last_known_addr { host: "127.4.129.254" port: 36341 } } peers { permanent_uuid: "0920946e0a6b490a83d891822679cd0c" member_type: VOTER last_known_addr { host: "127.4.129.253" port: 39681 } } peers { permanent_uuid: "fe8426faeb6a4fc898c723f1e66f1298" member_type: VOTER last_known_addr { host: "127.4.129.252" port: 39793 } }
16:30:34.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.361187 4859 sys_catalog.cc:455] T 00000000000000000000000000000000 P fe8426faeb6a4fc898c723f1e66f1298 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cc68485f0c2844df88f7839fa23f03f0" member_type: VOTER last_known_addr { host: "127.4.129.254" port: 36341 } } peers { permanent_uuid: "0920946e0a6b490a83d891822679cd0c" member_type: VOTER last_known_addr { host: "127.4.129.253" port: 39681 } } peers { permanent_uuid: "fe8426faeb6a4fc898c723f1e66f1298" member_type: VOTER last_known_addr { host: "127.4.129.252" port: 39793 } } }
16:30:34.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.361948 4859 sys_catalog.cc:458] T 00000000000000000000000000000000 P fe8426faeb6a4fc898c723f1e66f1298 [sys.catalog]: This master's current role is: FOLLOWER
16:30:34.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.363150 4850 sys_catalog.cc:564] T 00000000000000000000000000000000 P fe8426faeb6a4fc898c723f1e66f1298 [sys.catalog]: configured and running, proceeding with master startup.
16:30:34.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:34.398679 4870 catalog_manager.cc:1558] T 00000000000000000000000000000000 P fe8426faeb6a4fc898c723f1e66f1298: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:30:34.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:34.399528 4870 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:30:34.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.390637 4772 master_runner.cc:186] Error getting master registration for 127.4.129.254:36341: OK, instance_id { permanent_uuid: "cc68485f0c2844df88f7839fa23f03f0" instance_seqno: 1790440230450212 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
16:30:34.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.428499 4772 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:30:34.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.514369 4859 raft_consensus.cc:482] T 00000000000000000000000000000000 P fe8426faeb6a4fc898c723f1e66f1298 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
16:30:34.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.514972 4859 raft_consensus.cc:504] T 00000000000000000000000000000000 P fe8426faeb6a4fc898c723f1e66f1298 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cc68485f0c2844df88f7839fa23f03f0" member_type: VOTER last_known_addr { host: "127.4.129.254" port: 36341 } } peers { permanent_uuid: "0920946e0a6b490a83d891822679cd0c" member_type: VOTER last_known_addr { host: "127.4.129.253" port: 39681 } } peers { permanent_uuid: "fe8426faeb6a4fc898c723f1e66f1298" member_type: VOTER last_known_addr { host: "127.4.129.252" port: 39793 } }
16:30:34.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.521437 4859 leader_election.cc:288] T 00000000000000000000000000000000 P fe8426faeb6a4fc898c723f1e66f1298 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers cc68485f0c2844df88f7839fa23f03f0 (127.4.129.254:36341), 0920946e0a6b490a83d891822679cd0c (127.4.129.253:39681)
16:30:34.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:34.604368 4804 tablet.cc:2369] T 00000000000000000000000000000000 P fe8426faeb6a4fc898c723f1e66f1298: Can't schedule compaction. Clean time has not been advanced past its initial value.
16:30:34.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.690285 4770 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.129.252" port: 39793 } attempt: 7
16:30:34.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.725147 4770 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0920946e0a6b490a83d891822679cd0c: Bootstrap starting.
16:30:34.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.732901 4770 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 0920946e0a6b490a83d891822679cd0c: Neither blocks nor log segments found. Creating new log.
16:30:34.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.735674 4770 log.cc:824] T 00000000000000000000000000000000 P 0920946e0a6b490a83d891822679cd0c: Log is configured to *not* fsync() on all Append() calls
16:30:34.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.748284 4770 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0920946e0a6b490a83d891822679cd0c: No bootstrap required, opened a new log
16:30:34.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.780930 4770 raft_consensus.cc:348] T 00000000000000000000000000000000 P 0920946e0a6b490a83d891822679cd0c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cc68485f0c2844df88f7839fa23f03f0" member_type: VOTER last_known_addr { host: "127.4.129.254" port: 36341 } } peers { permanent_uuid: "0920946e0a6b490a83d891822679cd0c" member_type: VOTER last_known_addr { host: "127.4.129.253" port: 39681 } } peers { permanent_uuid: "fe8426faeb6a4fc898c723f1e66f1298" member_type: VOTER last_known_addr { host: "127.4.129.252" port: 39793 } }
16:30:34.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.782145 4770 raft_consensus.cc:374] T 00000000000000000000000000000000 P 0920946e0a6b490a83d891822679cd0c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
16:30:34.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.783006 4770 raft_consensus.cc:729] T 00000000000000000000000000000000 P 0920946e0a6b490a83d891822679cd0c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0920946e0a6b490a83d891822679cd0c, State: Initialized, Role: FOLLOWER
16:30:34.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.784178 4770 consensus_queue.cc:260] T 00000000000000000000000000000000 P 0920946e0a6b490a83d891822679cd0c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cc68485f0c2844df88f7839fa23f03f0" member_type: VOTER last_known_addr { host: "127.4.129.254" port: 36341 } } peers { permanent_uuid: "0920946e0a6b490a83d891822679cd0c" member_type: VOTER last_known_addr { host: "127.4.129.253" port: 39681 } } peers { permanent_uuid: "fe8426faeb6a4fc898c723f1e66f1298" member_type: VOTER last_known_addr { host: "127.4.129.252" port: 39793 } }
16:30:34.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.789255 4770 sys_catalog.cc:564] T 00000000000000000000000000000000 P 0920946e0a6b490a83d891822679cd0c [sys.catalog]: configured and running, proceeding with master startup.
16:30:34.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.789454 4745 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "fe8426faeb6a4fc898c723f1e66f1298" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0920946e0a6b490a83d891822679cd0c" is_pre_election: true
16:30:34.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.790343 4745 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 0920946e0a6b490a83d891822679cd0c [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate fe8426faeb6a4fc898c723f1e66f1298 in term 0.
16:30:34.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.788359 4874 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0920946e0a6b490a83d891822679cd0c [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cc68485f0c2844df88f7839fa23f03f0" member_type: VOTER last_known_addr { host: "127.4.129.254" port: 36341 } } peers { permanent_uuid: "0920946e0a6b490a83d891822679cd0c" member_type: VOTER last_known_addr { host: "127.4.129.253" port: 39681 } } peers { permanent_uuid: "fe8426faeb6a4fc898c723f1e66f1298" member_type: VOTER last_known_addr { host: "127.4.129.252" port: 39793 } } }
16:30:34.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.791986 4797 leader_election.cc:302] T 00000000000000000000000000000000 P fe8426faeb6a4fc898c723f1e66f1298 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0920946e0a6b490a83d891822679cd0c, fe8426faeb6a4fc898c723f1e66f1298; no voters:
16:30:34.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.792678 4874 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0920946e0a6b490a83d891822679cd0c [sys.catalog]: This master's current role is: FOLLOWER
16:30:34.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.792963 4859 raft_consensus.cc:2781] T 00000000000000000000000000000000 P fe8426faeb6a4fc898c723f1e66f1298 [term 0 FOLLOWER]: Leader pre-election won for term 1
16:30:34.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.793282 4859 raft_consensus.cc:482] T 00000000000000000000000000000000 P fe8426faeb6a4fc898c723f1e66f1298 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
16:30:34.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.793591 4859 raft_consensus.cc:3037] T 00000000000000000000000000000000 P fe8426faeb6a4fc898c723f1e66f1298 [term 0 FOLLOWER]: Advancing to term 1
16:30:34.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.800642 4859 raft_consensus.cc:504] T 00000000000000000000000000000000 P fe8426faeb6a4fc898c723f1e66f1298 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cc68485f0c2844df88f7839fa23f03f0" member_type: VOTER last_known_addr { host: "127.4.129.254" port: 36341 } } peers { permanent_uuid: "0920946e0a6b490a83d891822679cd0c" member_type: VOTER last_known_addr { host: "127.4.129.253" port: 39681 } } peers { permanent_uuid: "fe8426faeb6a4fc898c723f1e66f1298" member_type: VOTER last_known_addr { host: "127.4.129.252" port: 39793 } }
16:30:34.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.803781 4745 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "fe8426faeb6a4fc898c723f1e66f1298" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0920946e0a6b490a83d891822679cd0c"
16:30:34.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.804345 4745 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 0920946e0a6b490a83d891822679cd0c [term 0 FOLLOWER]: Advancing to term 1
16:30:34.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.810974 4859 leader_election.cc:288] T 00000000000000000000000000000000 P fe8426faeb6a4fc898c723f1e66f1298 [CANDIDATE]: Term 1 election: Requested vote from peers cc68485f0c2844df88f7839fa23f03f0 (127.4.129.254:36341), 0920946e0a6b490a83d891822679cd0c (127.4.129.253:39681)
16:30:34.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.811827 4745 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 0920946e0a6b490a83d891822679cd0c [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate fe8426faeb6a4fc898c723f1e66f1298 in term 1.
16:30:34.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.813535 4797 leader_election.cc:302] T 00000000000000000000000000000000 P fe8426faeb6a4fc898c723f1e66f1298 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0920946e0a6b490a83d891822679cd0c, fe8426faeb6a4fc898c723f1e66f1298; no voters:
16:30:34.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.820640 4859 raft_consensus.cc:2781] T 00000000000000000000000000000000 P fe8426faeb6a4fc898c723f1e66f1298 [term 1 FOLLOWER]: Leader election won for term 1
16:30:34.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.822453 4859 raft_consensus.cc:686] T 00000000000000000000000000000000 P fe8426faeb6a4fc898c723f1e66f1298 [term 1 LEADER]: Becoming Leader. State: Replica: fe8426faeb6a4fc898c723f1e66f1298, State: Running, Role: LEADER
16:30:34.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.824765 4859 consensus_queue.cc:237] T 00000000000000000000000000000000 P fe8426faeb6a4fc898c723f1e66f1298 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cc68485f0c2844df88f7839fa23f03f0" member_type: VOTER last_known_addr { host: "127.4.129.254" port: 36341 } } peers { permanent_uuid: "0920946e0a6b490a83d891822679cd0c" member_type: VOTER last_known_addr { host: "127.4.129.253" port: 39681 } } peers { permanent_uuid: "fe8426faeb6a4fc898c723f1e66f1298" member_type: VOTER last_known_addr { host: "127.4.129.252" port: 39793 } }
16:30:34.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.864696 4875 sys_catalog.cc:455] T 00000000000000000000000000000000 P fe8426faeb6a4fc898c723f1e66f1298 [sys.catalog]: SysCatalogTable state changed. Reason: New leader fe8426faeb6a4fc898c723f1e66f1298. Latest consensus state: current_term: 1 leader_uuid: "fe8426faeb6a4fc898c723f1e66f1298" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cc68485f0c2844df88f7839fa23f03f0" member_type: VOTER last_known_addr { host: "127.4.129.254" port: 36341 } } peers { permanent_uuid: "0920946e0a6b490a83d891822679cd0c" member_type: VOTER last_known_addr { host: "127.4.129.253" port: 39681 } } peers { permanent_uuid: "fe8426faeb6a4fc898c723f1e66f1298" member_type: VOTER last_known_addr { host: "127.4.129.252" port: 39793 } } }
16:30:34.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.866416 4875 sys_catalog.cc:458] T 00000000000000000000000000000000 P fe8426faeb6a4fc898c723f1e66f1298 [sys.catalog]: This master's current role is: LEADER
16:30:34.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.885169 4882 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
16:30:34.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.917613 4882 catalog_manager.cc:1484] Initializing Kudu cluster ID...
16:30:34.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.940287 4888 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
16:30:34.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.965416 4745 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 0920946e0a6b490a83d891822679cd0c [term 1 FOLLOWER]: Refusing update from remote peer fe8426faeb6a4fc898c723f1e66f1298: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
16:30:34.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.969416 4875 consensus_queue.cc:1035] T 00000000000000000000000000000000 P fe8426faeb6a4fc898c723f1e66f1298 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0920946e0a6b490a83d891822679cd0c" member_type: VOTER last_known_addr { host: "127.4.129.253" port: 39681 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
16:30:35.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:35.006318 4724 tablet.cc:2369] T 00000000000000000000000000000000 P 0920946e0a6b490a83d891822679cd0c: Can't schedule compaction. Clean time has not been advanced past its initial value.
16:30:35.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:34.945600 4697 master_runner.cc:186] Error getting master registration for 127.4.129.254:36341: OK, instance_id { permanent_uuid: "cc68485f0c2844df88f7839fa23f03f0" instance_seqno: 1790440230450212 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
16:30:35.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:35.011358 4697 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:30:35.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:35.038445 4874 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0920946e0a6b490a83d891822679cd0c [sys.catalog]: SysCatalogTable state changed. Reason: New leader fe8426faeb6a4fc898c723f1e66f1298. Latest consensus state: current_term: 1 leader_uuid: "fe8426faeb6a4fc898c723f1e66f1298" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cc68485f0c2844df88f7839fa23f03f0" member_type: VOTER last_known_addr { host: "127.4.129.254" port: 36341 } } peers { permanent_uuid: "0920946e0a6b490a83d891822679cd0c" member_type: VOTER last_known_addr { host: "127.4.129.253" port: 39681 } } peers { permanent_uuid: "fe8426faeb6a4fc898c723f1e66f1298" member_type: VOTER last_known_addr { host: "127.4.129.252" port: 39793 } } }
16:30:35.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:35.040148 4874 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0920946e0a6b490a83d891822679cd0c [sys.catalog]: This master's current role is: FOLLOWER
16:30:35.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:35.050311 4859 sys_catalog.cc:455] T 00000000000000000000000000000000 P fe8426faeb6a4fc898c723f1e66f1298 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "fe8426faeb6a4fc898c723f1e66f1298" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cc68485f0c2844df88f7839fa23f03f0" member_type: VOTER last_known_addr { host: "127.4.129.254" port: 36341 } } peers { permanent_uuid: "0920946e0a6b490a83d891822679cd0c" member_type: VOTER last_known_addr { host: "127.4.129.253" port: 39681 } } peers { permanent_uuid: "fe8426faeb6a4fc898c723f1e66f1298" member_type: VOTER last_known_addr { host: "127.4.129.252" port: 39793 } } }
16:30:35.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:35.051138 4859 sys_catalog.cc:458] T 00000000000000000000000000000000 P fe8426faeb6a4fc898c723f1e66f1298 [sys.catalog]: This master's current role is: LEADER
16:30:35.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:35.056593 4882 catalog_manager.cc:1347] Generated new cluster ID: 062749db54c84263883823c6e002fe93
16:30:35.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:35.056890 4882 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
16:30:35.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:35.063503 4874 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0920946e0a6b490a83d891822679cd0c [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "fe8426faeb6a4fc898c723f1e66f1298" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cc68485f0c2844df88f7839fa23f03f0" member_type: VOTER last_known_addr { host: "127.4.129.254" port: 36341 } } peers { permanent_uuid: "0920946e0a6b490a83d891822679cd0c" member_type: VOTER last_known_addr { host: "127.4.129.253" port: 39681 } } peers { permanent_uuid: "fe8426faeb6a4fc898c723f1e66f1298" member_type: VOTER last_known_addr { host: "127.4.129.252" port: 39793 } } }
16:30:35.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:35.065402 4874 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0920946e0a6b490a83d891822679cd0c [sys.catalog]: This master's current role is: FOLLOWER
16:30:35.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:35.101845 4882 catalog_manager.cc:1370] Generated new certificate authority record
16:30:35.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:35.108487 4882 catalog_manager.cc:1504] Loading token signing keys...
16:30:35.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:35.120134 4695 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.129.252" port: 39793 } attempt: 7
16:30:35.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:35.154023 4882 catalog_manager.cc:5868] T 00000000000000000000000000000000 P fe8426faeb6a4fc898c723f1e66f1298: Generated new TSK 0
16:30:35.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:35.157043 4882 catalog_manager.cc:1514] Initializing in-progress tserver states...
16:30:35.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:35.178757 4695 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P cc68485f0c2844df88f7839fa23f03f0: Bootstrap starting.
16:30:35.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:35.186417 4695 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P cc68485f0c2844df88f7839fa23f03f0: Neither blocks nor log segments found. Creating new log.
16:30:35.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:35.188865 4695 log.cc:824] T 00000000000000000000000000000000 P cc68485f0c2844df88f7839fa23f03f0: Log is configured to *not* fsync() on all Append() calls
16:30:35.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:35.194864 4695 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P cc68485f0c2844df88f7839fa23f03f0: No bootstrap required, opened a new log
16:30:35.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:35.204591 4615 external_mini_cluster.cc:1326] Running /tmp/dist-test-task4YrI62/build/tsan/bin/kudu
16:30:35.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4YrI62/build/tsan/bin/kudu
16:30:35.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/ts-0/wal
16:30:35.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/ts-0/data
16:30:35.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:30:35.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:30:35.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:30:35.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:30:35.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:30:35.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:30:35.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/ts-0/logs
16:30:35.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/ts-0/data/info.pb
16:30:35.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:30:35.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:30:35.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:30:35.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:30:35.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:30:35.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:30:35.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:30:35.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:30:35.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
16:30:35.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:30:35.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.129.193:0
16:30:35.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.129.193
16:30:35.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.129.193
16:30:35.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:30:35.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.129.254:36341,127.4.129.253:39681,127.4.129.252:39793
16:30:35.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.129.212:43985
16:30:35.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:30:35.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:30:35.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
16:30:35.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:35.242870 4695 raft_consensus.cc:348] T 00000000000000000000000000000000 P cc68485f0c2844df88f7839fa23f03f0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cc68485f0c2844df88f7839fa23f03f0" member_type: VOTER last_known_addr { host: "127.4.129.254" port: 36341 } } peers { permanent_uuid: "0920946e0a6b490a83d891822679cd0c" member_type: VOTER last_known_addr { host: "127.4.129.253" port: 39681 } } peers { permanent_uuid: "fe8426faeb6a4fc898c723f1e66f1298" member_type: VOTER last_known_addr { host: "127.4.129.252" port: 39793 } }
16:30:35.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:35.244135 4695 raft_consensus.cc:374] T 00000000000000000000000000000000 P cc68485f0c2844df88f7839fa23f03f0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
16:30:35.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:35.244647 4695 raft_consensus.cc:729] T 00000000000000000000000000000000 P cc68485f0c2844df88f7839fa23f03f0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: cc68485f0c2844df88f7839fa23f03f0, State: Initialized, Role: FOLLOWER
16:30:35.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:35.245777 4695 consensus_queue.cc:260] T 00000000000000000000000000000000 P cc68485f0c2844df88f7839fa23f03f0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cc68485f0c2844df88f7839fa23f03f0" member_type: VOTER last_known_addr { host: "127.4.129.254" port: 36341 } } peers { permanent_uuid: "0920946e0a6b490a83d891822679cd0c" member_type: VOTER last_known_addr { host: "127.4.129.253" port: 39681 } } peers { permanent_uuid: "fe8426faeb6a4fc898c723f1e66f1298" member_type: VOTER last_known_addr { host: "127.4.129.252" port: 39793 } }
16:30:35.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:35.254714 4897 sys_catalog.cc:455] T 00000000000000000000000000000000 P cc68485f0c2844df88f7839fa23f03f0 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cc68485f0c2844df88f7839fa23f03f0" member_type: VOTER last_known_addr { host: "127.4.129.254" port: 36341 } } peers { permanent_uuid: "0920946e0a6b490a83d891822679cd0c" member_type: VOTER last_known_addr { host: "127.4.129.253" port: 39681 } } peers { permanent_uuid: "fe8426faeb6a4fc898c723f1e66f1298" member_type: VOTER last_known_addr { host: "127.4.129.252" port: 39793 } } }
16:30:35.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:35.260674 4897 sys_catalog.cc:458] T 00000000000000000000000000000000 P cc68485f0c2844df88f7839fa23f03f0 [sys.catalog]: This master's current role is: FOLLOWER
16:30:35.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:35.260874 4668 raft_consensus.cc:3037] T 00000000000000000000000000000000 P cc68485f0c2844df88f7839fa23f03f0 [term 0 FOLLOWER]: Advancing to term 1
16:30:35.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:35.261221 4670 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "fe8426faeb6a4fc898c723f1e66f1298" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "cc68485f0c2844df88f7839fa23f03f0" is_pre_election: true
16:30:35.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:35.261785 4669 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "fe8426faeb6a4fc898c723f1e66f1298" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "cc68485f0c2844df88f7839fa23f03f0"
16:30:35.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:35.260308 4695 sys_catalog.cc:564] T 00000000000000000000000000000000 P cc68485f0c2844df88f7839fa23f03f0 [sys.catalog]: configured and running, proceeding with master startup.
16:30:35.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:35.267158 4668 raft_consensus.cc:1261] T 00000000000000000000000000000000 P cc68485f0c2844df88f7839fa23f03f0 [term 1 FOLLOWER]: Refusing update from remote peer fe8426faeb6a4fc898c723f1e66f1298: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
16:30:35.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:35.269114 4859 consensus_queue.cc:1035] T 00000000000000000000000000000000 P fe8426faeb6a4fc898c723f1e66f1298 [LEADER]: Connected to new peer: Peer: permanent_uuid: "cc68485f0c2844df88f7839fa23f03f0" member_type: VOTER last_known_addr { host: "127.4.129.254" port: 36341 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
16:30:35.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:35.358429 4897 sys_catalog.cc:455] T 00000000000000000000000000000000 P cc68485f0c2844df88f7839fa23f03f0 [sys.catalog]: SysCatalogTable state changed. Reason: New leader fe8426faeb6a4fc898c723f1e66f1298. Latest consensus state: current_term: 1 leader_uuid: "fe8426faeb6a4fc898c723f1e66f1298" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cc68485f0c2844df88f7839fa23f03f0" member_type: VOTER last_known_addr { host: "127.4.129.254" port: 36341 } } peers { permanent_uuid: "0920946e0a6b490a83d891822679cd0c" member_type: VOTER last_known_addr { host: "127.4.129.253" port: 39681 } } peers { permanent_uuid: "fe8426faeb6a4fc898c723f1e66f1298" member_type: VOTER last_known_addr { host: "127.4.129.252" port: 39793 } } }
16:30:35.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:35.359108 4897 sys_catalog.cc:458] T 00000000000000000000000000000000 P cc68485f0c2844df88f7839fa23f03f0 [sys.catalog]: This master's current role is: FOLLOWER
16:30:35.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:35.360984 4903 sys_catalog.cc:455] T 00000000000000000000000000000000 P cc68485f0c2844df88f7839fa23f03f0 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "fe8426faeb6a4fc898c723f1e66f1298" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cc68485f0c2844df88f7839fa23f03f0" member_type: VOTER last_known_addr { host: "127.4.129.254" port: 36341 } } peers { permanent_uuid: "0920946e0a6b490a83d891822679cd0c" member_type: VOTER last_known_addr { host: "127.4.129.253" port: 39681 } } peers { permanent_uuid: "fe8426faeb6a4fc898c723f1e66f1298" member_type: VOTER last_known_addr { host: "127.4.129.252" port: 39793 } } }
16:30:35.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:35.361737 4903 sys_catalog.cc:458] T 00000000000000000000000000000000 P cc68485f0c2844df88f7839fa23f03f0 [sys.catalog]: This master's current role is: FOLLOWER
16:30:35.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:35.430915 4913 catalog_manager.cc:1259] Loaded cluster ID: 062749db54c84263883823c6e002fe93
16:30:35.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:35.431684 4913 catalog_manager.cc:1552] T 00000000000000000000000000000000 P cc68485f0c2844df88f7839fa23f03f0: loading cluster ID for follower catalog manager: success
16:30:35.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:35.439879 4913 catalog_manager.cc:1574] T 00000000000000000000000000000000 P cc68485f0c2844df88f7839fa23f03f0: acquiring CA information for follower catalog manager: success
16:30:35.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:35.446311 4913 catalog_manager.cc:1602] T 00000000000000000000000000000000 P cc68485f0c2844df88f7839fa23f03f0: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
16:30:35.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:35.740443 4896 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:30:35.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:35.741091 4896 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:30:35.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:35.741878 4896 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:30:35.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:35.797806 4896 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:30:35.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:35.799340 4896 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.4.129.193
16:30:35.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:35.816527 4859 sys_catalog.cc:455] T 00000000000000000000000000000000 P fe8426faeb6a4fc898c723f1e66f1298 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "fe8426faeb6a4fc898c723f1e66f1298" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cc68485f0c2844df88f7839fa23f03f0" member_type: VOTER last_known_addr { host: "127.4.129.254" port: 36341 } } peers { permanent_uuid: "0920946e0a6b490a83d891822679cd0c" member_type: VOTER last_known_addr { host: "127.4.129.253" port: 39681 } } peers { permanent_uuid: "fe8426faeb6a4fc898c723f1e66f1298" member_type: VOTER last_known_addr { host: "127.4.129.252" port: 39793 } } }
16:30:35.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:35.817291 4859 sys_catalog.cc:458] T 00000000000000000000000000000000 P fe8426faeb6a4fc898c723f1e66f1298 [sys.catalog]: This master's current role is: LEADER
16:30:35.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:35.863670 4896 tablet_server_runner.cc:78] Tablet server non-default flags:
16:30:35.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:30:35.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.129.212:43985
16:30:35.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:30:35.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:30:35.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/ts-0/data
16:30:35.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/ts-0/wal
16:30:35.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:30:35.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:30:35.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.129.193:0
16:30:35.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:30:35.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:30:35.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:30:35.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/ts-0/data/info.pb
16:30:35.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.129.193
16:30:35.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:30:35.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.129.254:36341,127.4.129.253:39681,127.4.129.252:39793
16:30:35.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:30:35.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:30:35.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:30:35.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:30:35.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:30:35.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.129.193
16:30:35.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/ts-0/logs
16:30:35.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:30:35.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:30:35.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:30:35.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
16:30:35.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
16:30:35.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
16:30:35.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:30:35.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 15:11:41 UTC on ccycloud.quasar-yukscv.root.comops.site
16:30:35.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4674342
16:30:35.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:30:35.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:35.865329 4896 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:30:35.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:35.867487 4896 file_cache.cc:493] Constructed file cache file cache with capacity 419430
16:30:35.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:35.894187 4922 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:30:35.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:35.966943 4888 catalog_manager.cc:1259] Loaded cluster ID: 062749db54c84263883823c6e002fe93
16:30:35.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:35.967571 4888 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 0920946e0a6b490a83d891822679cd0c: loading cluster ID for follower catalog manager: success
16:30:35.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:35.989079 4888 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 0920946e0a6b490a83d891822679cd0c: acquiring CA information for follower catalog manager: success
16:30:36.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:36.008355 4888 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 0920946e0a6b490a83d891822679cd0c: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
16:30:38.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:37.286790 4921 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 4896
16:30:38.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:38.771646 4921 kernel_stack_watchdog.cc:198] Thread 4896 stuck at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:637 for 388ms:
16:30:38.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
16:30:38.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
16:30:38.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
16:30:38.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
16:30:38.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:35.896418 4923 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:30:38.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:38.773447 4896 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 2.877s user 0.598s sys 0.953s
16:30:38.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:38.773770 4896 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 2.878s user 0.598s sys 0.953s
16:30:38.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:38.774149 4925 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:30:38.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:38.777094 4924 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2877 milliseconds
16:30:38.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:38.777174 4896 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:30:38.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:38.778677 4896 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:30:38.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:38.781312 4896 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:30:38.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:38.783380 4896 hybrid_clock.cc:648] HybridClock initialized: now 1790440238783308 us; error 73 us; skew 500 ppm
16:30:38.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:38.784404 4896 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:30:38.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:38.805488 4896 webserver.cc:460] Webserver started at http://127.4.129.193:41251/ using document root <none> and password file <none>
16:30:38.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:38.806994 4896 fs_manager.cc:359] Metadata directory not provided
16:30:38.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:38.807291 4896 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:30:38.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:38.807840 4896 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
16:30:38.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:38.815460 4896 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/ts-0/data/instance:
16:30:38.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "184d1698982043abb38ee1a588a7f450"
16:30:38.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:30:38 on dist-test-slave-lbns"
16:30:38.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:38.817381 4896 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/ts-0/wal/instance:
16:30:38.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "184d1698982043abb38ee1a588a7f450"
16:30:38.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:30:38 on dist-test-slave-lbns"
16:30:38.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:38.828118 4896 fs_manager.cc:691] Time spent creating directory manager: real 0.010s user 0.007s sys 0.002s
16:30:38.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:38.836112 4934 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:30:38.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:38.837460 4896 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.001s sys 0.003s
16:30:38.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:38.837821 4896 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/ts-0/data,/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/ts-0/wal
16:30:38.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "184d1698982043abb38ee1a588a7f450"
16:30:38.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:30:38 on dist-test-slave-lbns"
16:30:38.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:38.838249 4896 fs_report.cc:383] FS layout report
16:30:38.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:30:38.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/ts-0/wal
16:30:38.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/ts-0/wal
16:30:38.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/ts-0/data/data
16:30:38.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:30:38.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:30:38.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:30:38.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:30:38.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:30:38.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:30:38.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:30:38.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:30:38.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:30:38.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:30:38.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:30:38.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:38.893980 4896 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
16:30:38.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:38.895934 4896 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:30:38.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:38.896502 4896 kserver.cc:163] Server-wide thread pool size limit: 3276
16:30:38.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:38.899945 4896 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
16:30:38.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:38.905830 4896 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
16:30:38.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:38.906159 4896 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
16:30:38.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:38.906454 4896 ts_tablet_manager.cc:610] Registered 0 tablets
16:30:38.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:38.906862 4896 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.001s sys 0.000s
16:30:39.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:39.127144 4896 rpc_server.cc:307] RPC server started. Bound to: 127.4.129.193:40467
16:30:39.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:39.127295 5047 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.129.193:40467 every 8 connection(s)
16:30:39.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:39.130658 4896 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/ts-0/data/info.pb
16:30:39.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:39.140332 4615 external_mini_cluster.cc:1388] Started /tmp/dist-test-task4YrI62/build/tsan/bin/kudu as pid 4896
16:30:39.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:39.141042 4615 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/ts-0/wal/instance
16:30:39.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:39.194767 4615 external_mini_cluster.cc:1326] Running /tmp/dist-test-task4YrI62/build/tsan/bin/kudu
16:30:39.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4YrI62/build/tsan/bin/kudu
16:30:39.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/ts-1/wal
16:30:39.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/ts-1/data
16:30:39.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:30:39.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:30:39.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:30:39.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:30:39.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:30:39.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:30:39.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/ts-1/logs
16:30:39.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/ts-1/data/info.pb
16:30:39.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:30:39.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:30:39.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:30:39.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:30:39.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:30:39.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:30:39.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:30:39.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:30:39.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
16:30:39.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:30:39.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.129.194:0
16:30:39.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.129.194
16:30:39.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.129.194
16:30:39.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:30:39.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.129.254:36341,127.4.129.253:39681,127.4.129.252:39793
16:30:39.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.129.212:43985
16:30:39.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:30:39.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:30:39.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
16:30:39.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:39.223676 5052 heartbeater.cc:347] Connected to a master server at 127.4.129.254:36341
16:30:39.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:39.224202 5052 heartbeater.cc:464] Registering TS with master...
16:30:39.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:39.225428 5052 heartbeater.cc:511] Master 127.4.129.254:36341 requested a full tablet report, sending...
16:30:39.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:39.229321 4660 ts_manager.cc:194] Registered new tserver with Master: 184d1698982043abb38ee1a588a7f450 (127.4.129.193:40467)
16:30:39.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:39.233908 5048 heartbeater.cc:347] Connected to a master server at 127.4.129.252:39793
16:30:39.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:39.234331 5048 heartbeater.cc:464] Registering TS with master...
16:30:39.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:39.235456 5048 heartbeater.cc:511] Master 127.4.129.252:39793 requested a full tablet report, sending...
16:30:39.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:39.238333 4815 ts_manager.cc:194] Registered new tserver with Master: 184d1698982043abb38ee1a588a7f450 (127.4.129.193:40467)
16:30:39.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:39.241212 4815 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.4.129.193:50751
16:30:39.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:39.256544 5049 heartbeater.cc:347] Connected to a master server at 127.4.129.253:39681
16:30:39.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:39.256940 5049 heartbeater.cc:464] Registering TS with master...
16:30:39.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:39.257691 5049 heartbeater.cc:511] Master 127.4.129.253:39681 requested a full tablet report, sending...
16:30:39.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:39.260787 4735 ts_manager.cc:194] Registered new tserver with Master: 184d1698982043abb38ee1a588a7f450 (127.4.129.193:40467)
16:30:39.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:39.580688 5058 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:30:39.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:39.581329 5058 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:30:39.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:39.582044 5058 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:30:39.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:39.622761 5058 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:30:39.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:39.623723 5058 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.4.129.194
16:30:39.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:39.660980 5058 tablet_server_runner.cc:78] Tablet server non-default flags:
16:30:39.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:30:39.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.129.212:43985
16:30:39.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:30:39.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:30:39.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/ts-1/data
16:30:39.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/ts-1/wal
16:30:39.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:30:39.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:30:39.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.129.194:0
16:30:39.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:30:39.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:30:39.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:30:39.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/ts-1/data/info.pb
16:30:39.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.129.194
16:30:39.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:30:39.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.129.254:36341,127.4.129.253:39681,127.4.129.252:39793
16:30:39.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:30:39.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:30:39.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:30:39.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:30:39.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:30:39.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.129.194
16:30:39.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/ts-1/logs
16:30:39.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:30:39.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:30:39.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:30:39.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
16:30:39.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
16:30:39.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
16:30:39.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:30:39.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 15:11:41 UTC on ccycloud.quasar-yukscv.root.comops.site
16:30:39.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4674342
16:30:39.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:30:39.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:39.662868 5058 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:30:39.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:39.665868 5058 file_cache.cc:493] Constructed file cache file cache with capacity 419430
16:30:39.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:39.695539 5064 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:30:40.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:40.246915 5048 heartbeater.cc:503] Master 127.4.129.252:39793 was elected leader, sending a full tablet report...
16:30:41.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:41.088073 5063 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 5058
16:30:41.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:41.274158 5063 kernel_stack_watchdog.cc:198] Thread 5058 stuck at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:637 for 387ms:
16:30:41.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
16:30:41.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
16:30:41.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
16:30:41.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
16:30:41.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:41.274950 5058 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 1.577s user 0.464s sys 0.840s
16:30:41.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:39.698879 5065 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:30:41.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:41.275312 5058 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 1.577s user 0.464s sys 0.840s
16:30:41.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:41.277456 5067 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:30:41.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:41.282675 5066 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1573 milliseconds
16:30:41.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:41.282673 5058 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:30:41.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:41.283994 5058 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:30:41.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:41.286018 5058 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:30:41.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:41.287420 5058 hybrid_clock.cc:648] HybridClock initialized: now 1790440241287390 us; error 44 us; skew 500 ppm
16:30:41.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:41.288338 5058 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:30:41.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:41.294834 5058 webserver.cc:460] Webserver started at http://127.4.129.194:45611/ using document root <none> and password file <none>
16:30:41.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:41.295710 5058 fs_manager.cc:359] Metadata directory not provided
16:30:41.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:41.295889 5058 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:30:41.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:41.296237 5058 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
16:30:41.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:41.300916 5058 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/ts-1/data/instance:
16:30:41.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2c7bd74b20744295b670a87fa81dcead"
16:30:41.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:30:41 on dist-test-slave-lbns"
16:30:41.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:41.301970 5058 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/ts-1/wal/instance:
16:30:41.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2c7bd74b20744295b670a87fa81dcead"
16:30:41.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:30:41 on dist-test-slave-lbns"
16:30:41.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:41.308884 5058 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.009s sys 0.001s
16:30:41.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:41.314443 5075 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:30:41.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:41.315675 5058 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.001s sys 0.003s
16:30:41.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:41.316079 5058 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/ts-1/data,/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/ts-1/wal
16:30:41.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2c7bd74b20744295b670a87fa81dcead"
16:30:41.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:30:41 on dist-test-slave-lbns"
16:30:41.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:41.316424 5058 fs_report.cc:383] FS layout report
16:30:41.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:30:41.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/ts-1/wal
16:30:41.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/ts-1/wal
16:30:41.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/ts-1/data/data
16:30:41.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:30:41.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:30:41.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:30:41.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:30:41.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:30:41.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:30:41.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:30:41.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:30:41.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:30:41.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:30:41.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:30:41.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:41.387102 5058 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
16:30:41.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:41.388725 5058 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:30:41.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:41.389202 5058 kserver.cc:163] Server-wide thread pool size limit: 3276
16:30:41.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:41.392441 5058 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
16:30:41.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:41.396768 5058 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
16:30:41.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:41.397010 5058 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
16:30:41.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:41.397313 5058 ts_tablet_manager.cc:610] Registered 0 tablets
16:30:41.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:41.397521 5058 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
16:30:41.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:41.553308 5058 rpc_server.cc:307] RPC server started. Bound to: 127.4.129.194:41057
16:30:41.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:41.553411 5187 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.129.194:41057 every 8 connection(s)
16:30:41.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:41.556020 5058 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/ts-1/data/info.pb
16:30:41.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:41.560482 4615 external_mini_cluster.cc:1388] Started /tmp/dist-test-task4YrI62/build/tsan/bin/kudu as pid 5058
16:30:41.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:41.560964 4615 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/ts-1/wal/instance
16:30:41.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:41.577204 4615 external_mini_cluster.cc:1326] Running /tmp/dist-test-task4YrI62/build/tsan/bin/kudu
16:30:41.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4YrI62/build/tsan/bin/kudu
16:30:41.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/ts-2/wal
16:30:41.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/ts-2/data
16:30:41.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:30:41.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:30:41.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:30:41.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:30:41.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:30:41.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:30:41.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/ts-2/logs
16:30:41.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/ts-2/data/info.pb
16:30:41.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:30:41.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:30:41.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:30:41.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:30:41.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:30:41.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:30:41.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:30:41.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:30:41.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
16:30:41.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:30:41.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.129.195:0
16:30:41.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.129.195
16:30:41.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.129.195
16:30:41.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:30:41.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.129.254:36341,127.4.129.253:39681,127.4.129.252:39793
16:30:41.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.129.212:43985
16:30:41.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:30:41.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:30:41.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
16:30:41.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:41.615741 5188 heartbeater.cc:347] Connected to a master server at 127.4.129.252:39793
16:30:41.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:41.616262 5188 heartbeater.cc:464] Registering TS with master...
16:30:41.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:41.617460 5188 heartbeater.cc:511] Master 127.4.129.252:39793 requested a full tablet report, sending...
16:30:41.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:41.620297 4815 ts_manager.cc:194] Registered new tserver with Master: 2c7bd74b20744295b670a87fa81dcead (127.4.129.194:41057)
16:30:41.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:41.622573 4815 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.4.129.194:44395
16:30:41.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:41.624547 5189 heartbeater.cc:347] Connected to a master server at 127.4.129.253:39681
16:30:41.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:41.624882 5189 heartbeater.cc:464] Registering TS with master...
16:30:41.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:41.626230 5189 heartbeater.cc:511] Master 127.4.129.253:39681 requested a full tablet report, sending...
16:30:41.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:41.629271 4735 ts_manager.cc:194] Registered new tserver with Master: 2c7bd74b20744295b670a87fa81dcead (127.4.129.194:41057)
16:30:41.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:41.632215 5192 heartbeater.cc:347] Connected to a master server at 127.4.129.254:36341
16:30:41.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:41.632470 5192 heartbeater.cc:464] Registering TS with master...
16:30:41.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:41.632975 5192 heartbeater.cc:511] Master 127.4.129.254:36341 requested a full tablet report, sending...
16:30:41.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:41.635104 4660 ts_manager.cc:194] Registered new tserver with Master: 2c7bd74b20744295b670a87fa81dcead (127.4.129.194:41057)
16:30:41.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:41.925138 5195 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:30:41.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:41.925748 5195 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:30:41.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:41.926348 5195 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:30:41.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:41.958288 5195 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:30:41.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:41.959292 5195 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.4.129.195
16:30:41.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:41.994611 5195 tablet_server_runner.cc:78] Tablet server non-default flags:
16:30:41.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:30:41.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.129.212:43985
16:30:41.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:30:41.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:30:41.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/ts-2/data
16:30:41.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/ts-2/wal
16:30:41.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:30:41.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:30:41.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.129.195:0
16:30:41.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:30:41.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:30:41.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:30:41.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/ts-2/data/info.pb
16:30:41.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.129.195
16:30:41.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:30:41.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.129.254:36341,127.4.129.253:39681,127.4.129.252:39793
16:30:41.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:30:41.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:30:41.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:30:41.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:30:41.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:30:41.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.129.195
16:30:41.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/ts-2/logs
16:30:41.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:30:41.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:30:41.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:30:41.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
16:30:41.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
16:30:41.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
16:30:41.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:30:41.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 15:11:41 UTC on ccycloud.quasar-yukscv.root.comops.site
16:30:41.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4674342
16:30:41.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:30:41.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:41.996196 5195 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:30:41.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:41.998334 5195 file_cache.cc:493] Constructed file cache file cache with capacity 419430
16:30:42.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:42.020713 5207 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:30:42.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:42.627542 5188 heartbeater.cc:503] Master 127.4.129.252:39793 was elected leader, sending a full tablet report...
16:30:43.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:42.021147 5204 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:30:43.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:42.020756 5205 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:30:43.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:43.385341 5206 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1362 milliseconds
16:30:43.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:43.385450 5195 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:30:43.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:43.386825 5195 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:30:43.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:43.389572 5195 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:30:43.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:43.391040 5195 hybrid_clock.cc:648] HybridClock initialized: now 1790440243390983 us; error 63 us; skew 500 ppm
16:30:43.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:43.391839 5195 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:30:43.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:43.401849 5195 webserver.cc:460] Webserver started at http://127.4.129.195:43653/ using document root <none> and password file <none>
16:30:43.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:43.402817 5195 fs_manager.cc:359] Metadata directory not provided
16:30:43.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:43.403008 5195 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:30:43.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:43.403381 5195 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
16:30:43.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:43.408387 5195 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/ts-2/data/instance:
16:30:43.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "38329420bcbc4614ba001a0550c9213e"
16:30:43.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:30:43 on dist-test-slave-lbns"
16:30:43.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:43.409474 5195 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/ts-2/wal/instance:
16:30:43.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "38329420bcbc4614ba001a0550c9213e"
16:30:43.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:30:43 on dist-test-slave-lbns"
16:30:43.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:43.417084 5195 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.004s
16:30:43.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:43.431244 5215 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:30:43.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:43.432447 5195 fs_manager.cc:725] Time spent opening block manager: real 0.011s user 0.004s sys 0.004s
16:30:43.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:43.432857 5195 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/ts-2/data,/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/ts-2/wal
16:30:43.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "38329420bcbc4614ba001a0550c9213e"
16:30:43.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:30:43 on dist-test-slave-lbns"
16:30:43.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:43.433251 5195 fs_report.cc:383] FS layout report
16:30:43.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:30:43.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/ts-2/wal
16:30:43.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/ts-2/wal
16:30:43.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/ts-2/data/data
16:30:43.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:30:43.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:30:43.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:30:43.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:30:43.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:30:43.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:30:43.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:30:43.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:30:43.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:30:43.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:30:43.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:30:43.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:43.479952 5195 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
16:30:43.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:43.481536 5195 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:30:43.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:43.481985 5195 kserver.cc:163] Server-wide thread pool size limit: 3276
16:30:43.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:43.484817 5195 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
16:30:43.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:43.489326 5195 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
16:30:43.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:43.489554 5195 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
16:30:43.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:43.489773 5195 ts_tablet_manager.cc:610] Registered 0 tablets
16:30:43.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:43.489919 5195 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
16:30:43.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:43.651985 5195 rpc_server.cc:307] RPC server started. Bound to: 127.4.129.195:38695
16:30:43.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:43.652099 5327 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.129.195:38695 every 8 connection(s)
16:30:43.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:43.654459 5195 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/ts-2/data/info.pb
16:30:43.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:43.657521 4615 external_mini_cluster.cc:1388] Started /tmp/dist-test-task4YrI62/build/tsan/bin/kudu as pid 5195
16:30:43.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:43.658017 4615 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster4148787256493774260/ts-2/wal/instance
16:30:43.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:43.709067 5330 heartbeater.cc:347] Connected to a master server at 127.4.129.254:36341
16:30:43.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:43.709589 5330 heartbeater.cc:464] Registering TS with master...
16:30:43.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:43.710328 5329 heartbeater.cc:347] Connected to a master server at 127.4.129.253:39681
16:30:43.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:43.711156 5329 heartbeater.cc:464] Registering TS with master...
16:30:43.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:43.711278 5328 heartbeater.cc:347] Connected to a master server at 127.4.129.252:39793
16:30:43.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:43.711457 5330 heartbeater.cc:511] Master 127.4.129.254:36341 requested a full tablet report, sending...
16:30:43.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:43.711654 5328 heartbeater.cc:464] Registering TS with master...
16:30:43.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:43.712267 5329 heartbeater.cc:511] Master 127.4.129.253:39681 requested a full tablet report, sending...
16:30:43.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:43.712352 5328 heartbeater.cc:511] Master 127.4.129.252:39793 requested a full tablet report, sending...
16:30:43.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:43.714679 4660 ts_manager.cc:194] Registered new tserver with Master: 38329420bcbc4614ba001a0550c9213e (127.4.129.195:38695)
16:30:43.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:43.714998 4735 ts_manager.cc:194] Registered new tserver with Master: 38329420bcbc4614ba001a0550c9213e (127.4.129.195:38695)
16:30:43.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:43.714995 4815 ts_manager.cc:194] Registered new tserver with Master: 38329420bcbc4614ba001a0550c9213e (127.4.129.195:38695)
16:30:43.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:43.717139 4815 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.4.129.195:39111
16:30:43.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:43.728821 4615 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
16:30:43.740 [DEBUG - main] (MiniKuduCluster.java:195) Response:
16:30:43.742 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
16:30:43.754 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.4.129.254"
port: 36341
}
bound_http_address {
host: "127.4.129.254"
port: 45147
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.4.129.253"
port: 39681
}
bound_http_address {
host: "127.4.129.253"
port: 35981
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.4.129.252"
port: 39793
}
bound_http_address {
host: "127.4.129.252"
port: 37457
}
}
}
16:30:43.760 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
16:30:43.764 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.4.129.193"
port: 40467
}
bound_http_address {
host: "127.4.129.193"
port: 41251
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.4.129.194"
port: 41057
}
bound_http_address {
host: "127.4.129.194"
port: 45611
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.4.129.195"
port: 38695
}
bound_http_address {
host: "127.4.129.195"
port: 43653
}
}
}
16:30:43.764 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
16:30:43.967 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
16:30:44.098 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task4YrI62/test-tmp
16:30:44.099 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
16:30:44.100 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task4YrI62/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
16:30:44.103 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
16:30:44.507 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:30:44.508 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
16:30:44.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:44.511159 5339 mini_oidc.cc:152] Starting JWKS server
16:30:44.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:44.514968 5339 webserver.cc:460] Webserver started at http://127.0.0.1:37491/ using document root <none> and password file <none>
16:30:44.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:44.515381 5339 mini_oidc.cc:189] Starting OIDC Discovery server
16:30:44.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:44.517103 5339 webserver.cc:460] Webserver started at http://127.0.0.1:45239/ using document root <none> and password file <none>
16:30:44.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
16:30:44.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/krb5kdc/principal' for realm 'KRBTEST.COM',
16:30:44.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
16:30:44.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:30:44 dist-test-slave-lbns krb5kdc[5351](info): setting up network...
16:30:44.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
16:30:44.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:30:44 dist-test-slave-lbns krb5kdc[5351](info): set up 2 sockets
16:30:44.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:30:44 dist-test-slave-lbns krb5kdc[5351](info): commencing operation
16:30:44.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
16:30:44.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:44.722400 5328 heartbeater.cc:503] Master 127.4.129.252:39793 was elected leader, sending a full tablet report...
16:30:45.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:45.655616 5339 mini_kdc.cc:121] Time spent starting KDC: real 1.112s user 0.010s sys 0.010s
16:30:45.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
16:30:45.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
16:30:45.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
16:30:45.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
16:30:45.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
16:30:45.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
16:30:45.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
16:30:45.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
16:30:45.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
16:30:45.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
16:30:45.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/krb5kdc/test-user.keytab.
16:30:45.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/krb5kdc/test-user.keytab.
16:30:45.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:30:45 dist-test-slave-lbns krb5kdc[5351](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790440245, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:30:45.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
16:30:45.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-26T16:30:45Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
16:30:45.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-26T16:30:45Z Disabled control of system clock
16:30:45.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.5.54.254@KRBTEST.COM; defaulting to no policy
16:30:45.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:30:45.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.5.54.254@KRBTEST.COM" created.
16:30:45.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:30:45.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.5.54.254 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/krb5kdc/kudu_127.5.54.254.keytab.
16:30:45.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.5.54.254 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/krb5kdc/kudu_127.5.54.254.keytab.
16:30:45.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:45.876398 5339 external_mini_cluster.cc:1326] Running /tmp/dist-test-task4YrI62/build/tsan/bin/kudu
16:30:45.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4YrI62/build/tsan/bin/kudu
16:30:45.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-0/wal
16:30:45.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-0/data
16:30:45.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:30:45.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:30:45.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:30:45.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:30:45.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:30:45.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:30:45.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-0/logs
16:30:45.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-0/data/info.pb
16:30:45.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:30:45.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:30:45.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:30:45.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:30:45.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:30:45.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:30:45.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:30:45.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:30:45.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:30:45.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:30:45.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:30:45.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:30:45.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.5.54.254:39175
16:30:45.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.5.54.254
16:30:45.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:30:45.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/krb5kdc/kudu_127.5.54.254.keytab
16:30:45.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.5.54.254
16:30:45.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:30:45.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:30:45.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:30:45.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.5.54.212:38523
16:30:45.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:30:45.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:30:45.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:30:45.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:30:45.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.5.54.254:39175,127.5.54.253:36663,127.5.54.252:43033
16:30:45.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:30:45.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37491/jwks.json
16:30:45.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:30:46.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:46.214404 5365 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:30:46.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:46.215054 5365 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:30:46.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:46.215662 5365 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:30:46.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:46.247586 5365 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
16:30:46.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:46.247814 5365 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:30:46.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:46.248159 5365 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:30:46.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:46.248432 5365 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:30:46.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:46.248652 5365 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:30:46.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:46.248922 5365 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:30:46.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:46.249080 5365 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37491/jwks.json
16:30:46.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:46.287698 5365 master_runner.cc:386] Master server non-default flags:
16:30:46.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:30:46.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.5.54.212:38523
16:30:46.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:30:46.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:30:46.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-0/data
16:30:46.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-0/wal
16:30:46.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
16:30:46.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:30:46.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.5.54.254:39175,127.5.54.253:36663,127.5.54.252:43033
16:30:46.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/krb5kdc/kudu_127.5.54.254.keytab
16:30:46.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.5.54.254
16:30:46.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:30:46.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:30:46.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:30:46.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.5.54.254:39175
16:30:46.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:30:46.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:30:46.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:30:46.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37491/jwks.json
16:30:46.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:30:46.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:30:46.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:30:46.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:30:46.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:30:46.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-0/data/info.pb
16:30:46.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.5.54.254
16:30:46.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:30:46.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:30:46.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:30:46.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:30:46.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:30:46.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:30:46.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-0/logs
16:30:46.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:30:46.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:30:46.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:30:46.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:30:46.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
16:30:46.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
16:30:46.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:30:46.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 15:11:41 UTC on ccycloud.quasar-yukscv.root.comops.site
16:30:46.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4674342
16:30:46.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:30:46.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:46.288866 5365 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:30:46.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:46.289959 5365 file_cache.cc:493] Constructed file cache file cache with capacity 419430
16:30:46.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:46.311525 5372 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:30:47.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:46.311537 5371 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:30:47.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:46.311863 5374 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:30:47.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:46.314005 5365 server_base.cc:1034] running on GCE node
16:30:47.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:47.595412 5365 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:30:47.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:47.597908 5365 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:30:47.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:47.599324 5365 hybrid_clock.cc:648] HybridClock initialized: now 1790440247599300 us; error 75 us; skew 500 ppm
16:30:47.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:47.600112 5365 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:30:47.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:30:47 dist-test-slave-lbns krb5kdc[5351](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790440247, etypes {rep=17 tkt=17 ses=17}, kudu/127.5.54.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:30:47.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:47.607100 5365 init.cc:377] Logged in from keytab as kudu/127.5.54.254@KRBTEST.COM (short username kudu)
16:30:47.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:47.612854 5365 webserver.cc:460] Webserver started at http://127.5.54.254:36675/ using document root <none> and password file <none>
16:30:47.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:47.613819 5365 fs_manager.cc:359] Metadata directory not provided
16:30:47.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:47.614017 5365 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:30:47.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:47.614434 5365 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
16:30:47.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:47.619503 5365 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-0/data/instance:
16:30:47.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f364d2622ec34443a4604cce6270022d"
16:30:47.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:30:47 on dist-test-slave-lbns"
16:30:47.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:47.620887 5365 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-0/wal/instance:
16:30:47.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f364d2622ec34443a4604cce6270022d"
16:30:47.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:30:47 on dist-test-slave-lbns"
16:30:47.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:47.628671 5365 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.000s sys 0.009s
16:30:47.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:47.634668 5382 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:30:47.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:47.635933 5365 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
16:30:47.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:47.636247 5365 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-0/data,/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-0/wal
16:30:47.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f364d2622ec34443a4604cce6270022d"
16:30:47.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:30:47 on dist-test-slave-lbns"
16:30:47.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:47.636572 5365 fs_report.cc:383] FS layout report
16:30:47.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:30:47.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-0/wal
16:30:47.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-0/wal
16:30:47.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-0/data/data
16:30:47.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:30:47.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:30:47.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:30:47.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:30:47.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:30:47.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:30:47.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:30:47.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:30:47.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:30:47.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:30:47.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:30:47.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:47.686504 5365 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
16:30:47.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:47.692512 5365 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:30:47.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:47.693313 5365 kserver.cc:163] Server-wide thread pool size limit: 3276
16:30:47.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:47.783372 5365 rpc_server.cc:307] RPC server started. Bound to: 127.5.54.254:39175
16:30:47.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:47.783468 5433 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.5.54.254:39175 every 8 connection(s)
16:30:47.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:47.786074 5365 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-0/data/info.pb
16:30:47.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:47.791697 5434 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
16:30:47.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:47.796304 5339 external_mini_cluster.cc:1388] Started /tmp/dist-test-task4YrI62/build/tsan/bin/kudu as pid 5365
16:30:47.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:47.796715 5339 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-0/wal/instance
16:30:47.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:47.803817 5434 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.5.54.254" port: 39175 } has no permanent_uuid. Determining permanent_uuid...
16:30:47.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.5.54.253@KRBTEST.COM; defaulting to no policy
16:30:47.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:30:47.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.5.54.253@KRBTEST.COM" created.
16:30:47.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:30:47 dist-test-slave-lbns krb5kdc[5351](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790440247, etypes {rep=17 tkt=17 ses=17}, kudu/127.5.54.254@KRBTEST.COM for kudu/127.5.54.254@KRBTEST.COM
16:30:47.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:30:47.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.5.54.253 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/krb5kdc/kudu_127.5.54.253.keytab.
16:30:47.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.5.54.253 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/krb5kdc/kudu_127.5.54.253.keytab.
16:30:47.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:47.847941 5339 external_mini_cluster.cc:1326] Running /tmp/dist-test-task4YrI62/build/tsan/bin/kudu
16:30:47.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4YrI62/build/tsan/bin/kudu
16:30:47.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-1/wal
16:30:47.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-1/data
16:30:47.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:30:47.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:30:47.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:30:47.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:30:47.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:30:47.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:30:47.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-1/logs
16:30:47.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-1/data/info.pb
16:30:47.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:30:47.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:30:47.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:30:47.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:30:47.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:30:47.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:30:47.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:30:47.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:30:47.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:30:47.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:30:47.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:30:47.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:30:47.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.5.54.253:36663
16:30:47.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.5.54.253
16:30:47.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:30:47.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/krb5kdc/kudu_127.5.54.253.keytab
16:30:47.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.5.54.253
16:30:47.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:30:47.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:30:47.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:30:47.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.5.54.212:38523
16:30:47.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:30:47.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:30:47.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:30:47.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:30:47.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.5.54.254:39175,127.5.54.253:36663,127.5.54.252:43033
16:30:47.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:30:47.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37491/jwks.json
16:30:47.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:30:47.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:47.850566 5434 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.5.54.253" port: 36663 } has no permanent_uuid. Determining permanent_uuid...
16:30:47.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:47.852898 5386 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.5.54.253:36663: connect: Connection refused (error 111)
16:30:47.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:47.856951 5434 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.54.253:36663: Network error: Client connection negotiation failed: client connection to 127.5.54.253:36663: connect: Connection refused (error 111)
16:30:47.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:47.906572 5434 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.54.253" port: 36663 } attempt: 1
16:30:47.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:47.910846 5434 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.54.253:36663: Network error: Client connection negotiation failed: client connection to 127.5.54.253:36663: connect: Connection refused (error 111)
16:30:47.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:47.979583 5434 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.54.253" port: 36663 } attempt: 2
16:30:47.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:47.983681 5434 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.54.253:36663: Network error: Client connection negotiation failed: client connection to 127.5.54.253:36663: connect: Connection refused (error 111)
16:30:48.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:48.075119 5434 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.54.253" port: 36663 } attempt: 3
16:30:48.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:48.079108 5434 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.54.253:36663: Network error: Client connection negotiation failed: client connection to 127.5.54.253:36663: connect: Connection refused (error 111)
16:30:48.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:48.201099 5440 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:30:48.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:48.201764 5440 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:30:48.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:48.202762 5440 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:30:48.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:48.222453 5434 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.54.253" port: 36663 } attempt: 4
16:30:48.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:48.226040 5434 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.54.253:36663: Network error: Client connection negotiation failed: client connection to 127.5.54.253:36663: connect: Connection refused (error 111)
16:30:48.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:48.239544 5440 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
16:30:48.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:48.239777 5440 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:30:48.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:48.240033 5440 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:30:48.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:48.240226 5440 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:30:48.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:48.240515 5440 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:30:48.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:48.240725 5440 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:30:48.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:48.240865 5440 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37491/jwks.json
16:30:48.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:48.279706 5440 master_runner.cc:386] Master server non-default flags:
16:30:48.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:30:48.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.5.54.212:38523
16:30:48.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:30:48.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:30:48.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-1/data
16:30:48.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-1/wal
16:30:48.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
16:30:48.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:30:48.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.5.54.254:39175,127.5.54.253:36663,127.5.54.252:43033
16:30:48.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/krb5kdc/kudu_127.5.54.253.keytab
16:30:48.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.5.54.253
16:30:48.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:30:48.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:30:48.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:30:48.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.5.54.253:36663
16:30:48.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:30:48.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:30:48.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:30:48.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37491/jwks.json
16:30:48.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:30:48.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:30:48.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:30:48.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:30:48.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:30:48.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-1/data/info.pb
16:30:48.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.5.54.253
16:30:48.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:30:48.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:30:48.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:30:48.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:30:48.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:30:48.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:30:48.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-1/logs
16:30:48.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:30:48.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:30:48.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:30:48.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:30:48.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
16:30:48.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
16:30:48.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:30:48.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 15:11:41 UTC on ccycloud.quasar-yukscv.root.comops.site
16:30:48.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4674342
16:30:48.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:30:48.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:48.280808 5440 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:30:48.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:48.281989 5440 file_cache.cc:493] Constructed file cache file cache with capacity 419430
16:30:48.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:48.304173 5447 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:30:48.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:48.525478 5434 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.54.253" port: 36663 } attempt: 5
16:30:48.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:48.530628 5434 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.54.253:36663: Network error: Client connection negotiation failed: client connection to 127.5.54.253:36663: connect: Connection refused (error 111)
16:30:49.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:49.078056 5434 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.54.253" port: 36663 } attempt: 6
16:30:49.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:49.084477 5434 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.54.253:36663: Network error: Client connection negotiation failed: client connection to 127.5.54.253:36663: connect: Connection refused (error 111)
16:30:49.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:48.306649 5440 server_base.cc:1034] running on GCE node
16:30:49.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:48.304994 5448 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:30:49.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:48.304291 5450 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:30:49.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:49.585367 5440 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:30:49.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:49.588093 5440 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:30:49.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:49.589426 5440 hybrid_clock.cc:648] HybridClock initialized: now 1790440249589398 us; error 49 us; skew 500 ppm
16:30:49.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:49.590241 5440 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:30:49.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:30:49 dist-test-slave-lbns krb5kdc[5351](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790440249, etypes {rep=17 tkt=17 ses=17}, kudu/127.5.54.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:30:49.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:49.596993 5440 init.cc:377] Logged in from keytab as kudu/127.5.54.253@KRBTEST.COM (short username kudu)
16:30:49.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:49.602345 5440 webserver.cc:460] Webserver started at http://127.5.54.253:45717/ using document root <none> and password file <none>
16:30:49.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:49.603353 5440 fs_manager.cc:359] Metadata directory not provided
16:30:49.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:49.603600 5440 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:30:49.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:49.604128 5440 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
16:30:49.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:49.609194 5440 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-1/data/instance:
16:30:49.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e4f97fde47054172b47f501330f57c2a"
16:30:49.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:30:49 on dist-test-slave-lbns"
16:30:49.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:49.610478 5440 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-1/wal/instance:
16:30:49.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e4f97fde47054172b47f501330f57c2a"
16:30:49.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:30:49 on dist-test-slave-lbns"
16:30:49.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:49.618322 5440 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
16:30:49.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:49.624948 5463 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:30:49.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:49.626205 5440 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.004s
16:30:49.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:49.626495 5440 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-1/data,/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-1/wal
16:30:49.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e4f97fde47054172b47f501330f57c2a"
16:30:49.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:30:49 on dist-test-slave-lbns"
16:30:49.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:49.626897 5440 fs_report.cc:383] FS layout report
16:30:49.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:30:49.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-1/wal
16:30:49.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-1/wal
16:30:49.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-1/data/data
16:30:49.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:30:49.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:30:49.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:30:49.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:30:49.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:30:49.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:30:49.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:30:49.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:30:49.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:30:49.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:30:49.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:30:49.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:49.701236 5440 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
16:30:49.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:49.707147 5440 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:30:49.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:49.707561 5440 kserver.cc:163] Server-wide thread pool size limit: 3276
16:30:49.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:49.788126 5440 rpc_server.cc:307] RPC server started. Bound to: 127.5.54.253:36663
16:30:49.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:49.788172 5514 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.5.54.253:36663 every 8 connection(s)
16:30:49.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:49.791992 5440 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-1/data/info.pb
16:30:49.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:49.799422 5515 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
16:30:49.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:49.801244 5339 external_mini_cluster.cc:1388] Started /tmp/dist-test-task4YrI62/build/tsan/bin/kudu as pid 5440
16:30:49.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:49.801770 5339 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-1/wal/instance
16:30:49.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:49.817260 5515 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.5.54.254" port: 39175 } has no permanent_uuid. Determining permanent_uuid...
16:30:49.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.5.54.252@KRBTEST.COM; defaulting to no policy
16:30:49.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:30:49.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.5.54.252@KRBTEST.COM" created.
16:30:49.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:30:49 dist-test-slave-lbns krb5kdc[5351](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790440249, etypes {rep=17 tkt=17 ses=17}, kudu/127.5.54.253@KRBTEST.COM for kudu/127.5.54.254@KRBTEST.COM
16:30:49.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:30:49.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.5.54.252 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/krb5kdc/kudu_127.5.54.252.keytab.
16:30:49.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.5.54.252 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/krb5kdc/kudu_127.5.54.252.keytab.
16:30:49.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:49.852383 5339 external_mini_cluster.cc:1326] Running /tmp/dist-test-task4YrI62/build/tsan/bin/kudu
16:30:49.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4YrI62/build/tsan/bin/kudu
16:30:49.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-2/wal
16:30:49.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-2/data
16:30:49.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:30:49.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:30:49.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:30:49.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:30:49.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:30:49.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:30:49.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-2/logs
16:30:49.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-2/data/info.pb
16:30:49.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:30:49.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:30:49.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:30:49.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:30:49.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:30:49.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:30:49.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:30:49.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:30:49.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:30:49.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:30:49.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:30:49.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:30:49.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.5.54.252:43033
16:30:49.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.5.54.252
16:30:49.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:30:49.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/krb5kdc/kudu_127.5.54.252.keytab
16:30:49.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.5.54.252
16:30:49.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:30:49.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:30:49.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:30:49.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.5.54.212:38523
16:30:49.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:30:49.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:30:49.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:30:49.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:30:49.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.5.54.254:39175,127.5.54.253:36663,127.5.54.252:43033
16:30:49.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:30:49.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37491/jwks.json
16:30:49.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:30:49.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:49.860770 5515 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.5.54.253" port: 36663 } has no permanent_uuid. Determining permanent_uuid...
16:30:49.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:30:49 dist-test-slave-lbns krb5kdc[5351](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790440249, etypes {rep=17 tkt=17 ses=17}, kudu/127.5.54.253@KRBTEST.COM for kudu/127.5.54.253@KRBTEST.COM
16:30:49.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:49.892802 5515 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.5.54.252" port: 43033 } has no permanent_uuid. Determining permanent_uuid...
16:30:49.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:49.895205 5465 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.5.54.252:43033: connect: Connection refused (error 111)
16:30:49.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:49.900085 5515 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.54.252:43033: Network error: Client connection negotiation failed: client connection to 127.5.54.252:43033: connect: Connection refused (error 111)
16:30:49.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:49.949545 5515 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.54.252" port: 43033 } attempt: 1
16:30:49.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:49.954046 5515 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.54.252:43033: Network error: Client connection negotiation failed: client connection to 127.5.54.252:43033: connect: Connection refused (error 111)
16:30:50.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:50.022460 5515 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.54.252" port: 43033 } attempt: 2
16:30:50.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:50.027158 5515 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.54.252:43033: Network error: Client connection negotiation failed: client connection to 127.5.54.252:43033: connect: Connection refused (error 111)
16:30:50.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:50.118666 5515 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.54.252" port: 43033 } attempt: 3
16:30:50.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:50.123082 5515 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.54.252:43033: Network error: Client connection negotiation failed: client connection to 127.5.54.252:43033: connect: Connection refused (error 111)
16:30:50.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:50.145125 5434 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.54.253" port: 36663 } attempt: 7
16:30:50.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:30:50 dist-test-slave-lbns krb5kdc[5351](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790440247, etypes {rep=17 tkt=17 ses=17}, kudu/127.5.54.254@KRBTEST.COM for kudu/127.5.54.253@KRBTEST.COM
16:30:50.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:50.177999 5434 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.5.54.252" port: 43033 } has no permanent_uuid. Determining permanent_uuid...
16:30:50.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:50.184441 5434 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.54.252:43033: Network error: Client connection negotiation failed: client connection to 127.5.54.252:43033: connect: Connection refused (error 111)
16:30:50.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:50.212301 5521 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:30:50.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:50.212996 5521 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:30:50.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:50.213733 5521 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:30:50.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:50.242861 5434 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.54.252" port: 43033 } attempt: 1
16:30:50.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:50.246611 5434 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.54.252:43033: Network error: Client connection negotiation failed: client connection to 127.5.54.252:43033: connect: Connection refused (error 111)
16:30:50.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:50.247017 5521 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
16:30:50.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:50.247247 5521 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:30:50.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:50.247587 5521 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:30:50.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:50.247915 5521 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:30:50.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:50.248291 5521 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:30:50.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:50.248623 5521 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:30:50.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:50.248907 5521 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37491/jwks.json
16:30:50.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:50.266654 5515 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.54.252" port: 43033 } attempt: 4
16:30:50.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:50.270643 5515 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.54.252:43033: Network error: Client connection negotiation failed: client connection to 127.5.54.252:43033: connect: Connection refused (error 111)
16:30:50.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:50.291267 5521 master_runner.cc:386] Master server non-default flags:
16:30:50.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:30:50.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.5.54.212:38523
16:30:50.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:30:50.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:30:50.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-2/data
16:30:50.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-2/wal
16:30:50.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
16:30:50.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:30:50.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.5.54.254:39175,127.5.54.253:36663,127.5.54.252:43033
16:30:50.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/krb5kdc/kudu_127.5.54.252.keytab
16:30:50.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.5.54.252
16:30:50.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:30:50.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:30:50.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:30:50.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.5.54.252:43033
16:30:50.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:30:50.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:30:50.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:30:50.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37491/jwks.json
16:30:50.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:30:50.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:30:50.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:30:50.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:30:50.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:30:50.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-2/data/info.pb
16:30:50.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.5.54.252
16:30:50.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:30:50.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:30:50.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:30:50.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:30:50.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:30:50.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:30:50.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-2/logs
16:30:50.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:30:50.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:30:50.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:30:50.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:30:50.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
16:30:50.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
16:30:50.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:30:50.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 15:11:41 UTC on ccycloud.quasar-yukscv.root.comops.site
16:30:50.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4674342
16:30:50.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:30:50.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:50.292930 5521 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:30:50.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:50.294407 5521 file_cache.cc:493] Constructed file cache file cache with capacity 419430
16:30:50.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:50.315686 5531 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:30:50.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:50.328099 5434 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.54.252" port: 43033 } attempt: 2
16:30:50.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:50.331729 5434 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.54.252:43033: Network error: Client connection negotiation failed: client connection to 127.5.54.252:43033: connect: Connection refused (error 111)
16:30:50.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:50.417142 5434 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.54.252" port: 43033 } attempt: 3
16:30:50.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:50.420990 5434 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.54.252:43033: Network error: Client connection negotiation failed: client connection to 127.5.54.252:43033: connect: Connection refused (error 111)
16:30:50.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:50.561628 5434 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.54.252" port: 43033 } attempt: 4
16:30:50.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:50.566494 5434 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.54.252:43033: Network error: Client connection negotiation failed: client connection to 127.5.54.252:43033: connect: Connection refused (error 111)
16:30:50.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:50.570114 5515 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.54.252" port: 43033 } attempt: 5
16:30:50.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:50.574537 5515 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.54.252:43033: Network error: Client connection negotiation failed: client connection to 127.5.54.252:43033: connect: Connection refused (error 111)
16:30:50.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:50.849975 5434 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.54.252" port: 43033 } attempt: 5
16:30:50.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:50.855669 5434 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.54.252:43033: Network error: Client connection negotiation failed: client connection to 127.5.54.252:43033: connect: Connection refused (error 111)
16:30:51.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:51.122184 5515 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.54.252" port: 43033 } attempt: 6
16:30:51.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:51.134384 5515 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.54.252:43033: Network error: Client connection negotiation failed: client connection to 127.5.54.252:43033: connect: Connection refused (error 111)
16:30:51.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:51.408078 5434 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.54.252" port: 43033 } attempt: 6
16:30:51.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:51.414866 5434 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.54.252:43033: Network error: Client connection negotiation failed: client connection to 127.5.54.252:43033: connect: Connection refused (error 111)
16:30:51.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:50.316617 5532 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:30:51.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:50.317304 5534 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:30:51.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:51.617082 5533 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
16:30:51.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:51.617224 5521 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:30:51.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:51.621416 5521 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:30:51.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:51.624329 5521 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:30:51.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:51.625753 5521 hybrid_clock.cc:648] HybridClock initialized: now 1790440251625705 us; error 61 us; skew 500 ppm
16:30:51.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:51.626756 5521 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:30:51.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:30:51 dist-test-slave-lbns krb5kdc[5351](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790440251, etypes {rep=17 tkt=17 ses=17}, kudu/127.5.54.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:30:51.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:51.632854 5521 init.cc:377] Logged in from keytab as kudu/127.5.54.252@KRBTEST.COM (short username kudu)
16:30:51.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:51.638859 5521 webserver.cc:460] Webserver started at http://127.5.54.252:37353/ using document root <none> and password file <none>
16:30:51.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:51.639832 5521 fs_manager.cc:359] Metadata directory not provided
16:30:51.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:51.640038 5521 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:30:51.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:51.640422 5521 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
16:30:51.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:51.645849 5521 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-2/data/instance:
16:30:51.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7f2fd70ed2b0475184279df962be20a3"
16:30:51.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:30:51 on dist-test-slave-lbns"
16:30:51.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:51.647450 5521 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-2/wal/instance:
16:30:51.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7f2fd70ed2b0475184279df962be20a3"
16:30:51.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:30:51 on dist-test-slave-lbns"
16:30:51.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:51.657402 5521 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.005s sys 0.004s
16:30:51.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:51.665279 5547 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:30:51.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:51.667428 5521 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.005s sys 0.001s
16:30:51.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:51.667819 5521 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-2/data,/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-2/wal
16:30:51.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7f2fd70ed2b0475184279df962be20a3"
16:30:51.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:30:51 on dist-test-slave-lbns"
16:30:51.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:51.668224 5521 fs_report.cc:383] FS layout report
16:30:51.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:30:51.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-2/wal
16:30:51.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-2/wal
16:30:51.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-2/data/data
16:30:51.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:30:51.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:30:51.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:30:51.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:30:51.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:30:51.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:30:51.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:30:51.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:30:51.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:30:51.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:30:51.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:30:51.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:51.708205 5521 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
16:30:51.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:51.715523 5521 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:30:51.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:51.716091 5521 kserver.cc:163] Server-wide thread pool size limit: 3276
16:30:51.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:51.816857 5521 rpc_server.cc:307] RPC server started. Bound to: 127.5.54.252:43033
16:30:51.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:51.817000 5598 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.5.54.252:43033 every 8 connection(s)
16:30:51.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:51.819766 5521 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-2/data/info.pb
16:30:51.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:51.823424 5339 external_mini_cluster.cc:1388] Started /tmp/dist-test-task4YrI62/build/tsan/bin/kudu as pid 5521
16:30:51.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:51.823916 5339 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-2/wal/instance
16:30:51.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:51.826785 5599 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
16:30:51.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:51.840348 5599 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.5.54.254" port: 39175 } has no permanent_uuid. Determining permanent_uuid...
16:30:51.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:30:51 dist-test-slave-lbns krb5kdc[5351](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790440251, etypes {rep=17 tkt=17 ses=17}, kudu/127.5.54.252@KRBTEST.COM for kudu/127.5.54.254@KRBTEST.COM
16:30:51.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:30:51 dist-test-slave-lbns krb5kdc[5351](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790440245, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.5.54.252@KRBTEST.COM
16:30:51.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:51.880975 5599 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.5.54.253" port: 36663 } has no permanent_uuid. Determining permanent_uuid...
16:30:51.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:30:51 dist-test-slave-lbns krb5kdc[5351](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790440251, etypes {rep=17 tkt=17 ses=17}, kudu/127.5.54.252@KRBTEST.COM for kudu/127.5.54.253@KRBTEST.COM
16:30:51.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:30:51 dist-test-slave-lbns krb5kdc[5351](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790440245, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.5.54.253@KRBTEST.COM
16:30:51.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:51.905210 5599 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.5.54.252" port: 43033 } has no permanent_uuid. Determining permanent_uuid...
16:30:51.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:30:51 dist-test-slave-lbns krb5kdc[5351](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790440251, etypes {rep=17 tkt=17 ses=17}, kudu/127.5.54.252@KRBTEST.COM for kudu/127.5.54.252@KRBTEST.COM
16:30:51.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:30:51 dist-test-slave-lbns krb5kdc[5351](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790440245, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.5.54.254@KRBTEST.COM
16:30:51.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:51.940299 5599 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3: Bootstrap starting.
16:30:51.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:51.947105 5599 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3: Neither blocks nor log segments found. Creating new log.
16:30:51.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:51.948820 5339 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
16:30:51.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:51.949183 5599 log.cc:824] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3: Log is configured to *not* fsync() on all Append() calls
16:30:51.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:51.953837 5599 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3: No bootstrap required, opened a new log
16:30:51.956 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:30:51.957 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
16:30:51.971 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
16:30:51.971 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
16:30:51.975 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.5.54.254"
port: 39175
}
bound_http_address {
host: "127.5.54.254"
port: 36675
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.5.54.253"
port: 36663
}
bound_http_address {
host: "127.5.54.253"
port: 45717
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.5.54.252"
port: 43033
}
bound_http_address {
host: "127.5.54.252"
port: 37353
}
}
}
16:30:51.975 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
16:30:51.976 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:30:51.976 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
16:30:51.977 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
16:30:51.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:51.980278 5599 raft_consensus.cc:348] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f364d2622ec34443a4604cce6270022d" member_type: VOTER last_known_addr { host: "127.5.54.254" port: 39175 } } peers { permanent_uuid: "e4f97fde47054172b47f501330f57c2a" member_type: VOTER last_known_addr { host: "127.5.54.253" port: 36663 } } peers { permanent_uuid: "7f2fd70ed2b0475184279df962be20a3" member_type: VOTER last_known_addr { host: "127.5.54.252" port: 43033 } }
16:30:51.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:51.980962 5599 raft_consensus.cc:374] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
16:30:51.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:51.981259 5599 raft_consensus.cc:729] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7f2fd70ed2b0475184279df962be20a3, State: Initialized, Role: FOLLOWER
16:30:51.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:51.982266 5599 consensus_queue.cc:260] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f364d2622ec34443a4604cce6270022d" member_type: VOTER last_known_addr { host: "127.5.54.254" port: 39175 } } peers { permanent_uuid: "e4f97fde47054172b47f501330f57c2a" member_type: VOTER last_known_addr { host: "127.5.54.253" port: 36663 } } peers { permanent_uuid: "7f2fd70ed2b0475184279df962be20a3" member_type: VOTER last_known_addr { host: "127.5.54.252" port: 43033 } }
16:30:51.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:51.985066 5610 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f364d2622ec34443a4604cce6270022d" member_type: VOTER last_known_addr { host: "127.5.54.254" port: 39175 } } peers { permanent_uuid: "e4f97fde47054172b47f501330f57c2a" member_type: VOTER last_known_addr { host: "127.5.54.253" port: 36663 } } peers { permanent_uuid: "7f2fd70ed2b0475184279df962be20a3" member_type: VOTER last_known_addr { host: "127.5.54.252" port: 43033 } } }
16:30:51.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:51.985734 5610 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3 [sys.catalog]: This master's current role is: FOLLOWER
16:30:51.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:51.987243 5599 sys_catalog.cc:564] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3 [sys.catalog]: configured and running, proceeding with master startup.
16:30:52.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:30:52 dist-test-slave-lbns krb5kdc[5351](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790440252, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:30:52.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
16:30:52.012 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:30:52.014 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/krb5kdc/krb5cc
16:30:52.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:52.020795 5622 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:30:52.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:52.021627 5622 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:30:52.022 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
16:30:52.023 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/krb5kdc/krb5cc
16:30:52.025 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
16:30:52.029 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
16:30:52.031 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
16:30:52.031 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
16:30:52.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:52.012472 5521 master_runner.cc:186] Error getting master registration for 127.5.54.254:39175: OK, instance_id { permanent_uuid: "f364d2622ec34443a4604cce6270022d" instance_seqno: 1790440247757785 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
16:30:52.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:52.042232 5521 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:30:52.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:52.055763 5610 raft_consensus.cc:482] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
16:30:52.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:52.056325 5610 raft_consensus.cc:504] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f364d2622ec34443a4604cce6270022d" member_type: VOTER last_known_addr { host: "127.5.54.254" port: 39175 } } peers { permanent_uuid: "e4f97fde47054172b47f501330f57c2a" member_type: VOTER last_known_addr { host: "127.5.54.253" port: 36663 } } peers { permanent_uuid: "7f2fd70ed2b0475184279df962be20a3" member_type: VOTER last_known_addr { host: "127.5.54.252" port: 43033 } }
16:30:52.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:52.060400 5610 leader_election.cc:288] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f364d2622ec34443a4604cce6270022d (127.5.54.254:39175), e4f97fde47054172b47f501330f57c2a (127.5.54.253:36663)
16:30:52.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:30:52 dist-test-slave-lbns krb5kdc[5351](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790440252, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:30:52.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
16:30:52.073 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:30:52.073 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/krb5kdc/krb5cc
16:30:52.075 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
16:30:52.075 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.5.54.254:39175
16:30:52.077 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
16:30:52.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:52.078352 5339 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task4YrI62/build/tsan/bin/kudu with pid 5365
16:30:52.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:52.102243 5549 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv got EOF from 127.5.54.254:39175 (error 108)
16:30:52.106 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:30:52.106 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.5.54.252:43033
16:30:52.106 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
16:30:52.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:52.106863 5339 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task4YrI62/build/tsan/bin/kudu with pid 5521
16:30:52.130 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:30:52.130 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.5.54.253:36663
16:30:52.130 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
16:30:52.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:52.131253 5339 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task4YrI62/build/tsan/bin/kudu with pid 5440
16:30:52.154 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:30:52.154 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.5.54.254:39175
16:30:52.156 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
16:30:52.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:52.158736 5339 external_mini_cluster.cc:1326] Running /tmp/dist-test-task4YrI62/build/tsan/bin/kudu
16:30:52.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4YrI62/build/tsan/bin/kudu
16:30:52.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-0/wal
16:30:52.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-0/data
16:30:52.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:30:52.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:30:52.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:30:52.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:30:52.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:30:52.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:30:52.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-0/logs
16:30:52.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-0/data/info.pb
16:30:52.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:30:52.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:30:52.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:30:52.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:30:52.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:30:52.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:30:52.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:30:52.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:30:52.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:30:52.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:30:52.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:30:52.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:30:52.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.5.54.254:39175
16:30:52.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.5.54.254
16:30:52.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36675
16:30:52.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/krb5kdc/kudu_127.5.54.254.keytab
16:30:52.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.5.54.254
16:30:52.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:30:52.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:30:52.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:30:52.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.5.54.212:38523
16:30:52.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:30:52.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:30:52.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:30:52.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:30:52.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.5.54.254:39175,127.5.54.253:36663,127.5.54.252:43033
16:30:52.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:30:52.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37491/jwks.json
16:30:52.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:30:52.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:52.488924 5627 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:30:52.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:52.489661 5627 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:30:52.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:52.490407 5627 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:30:52.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:52.521675 5627 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
16:30:52.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:52.521976 5627 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:30:52.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:52.522372 5627 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:30:52.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:52.522732 5627 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:30:52.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:52.523057 5627 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:30:52.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:52.523376 5627 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:30:52.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:52.523619 5627 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37491/jwks.json
16:30:52.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:52.567143 5627 master_runner.cc:386] Master server non-default flags:
16:30:52.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:30:52.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.5.54.212:38523
16:30:52.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:30:52.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:30:52.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-0/data
16:30:52.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-0/wal
16:30:52.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
16:30:52.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:30:52.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.5.54.254:39175,127.5.54.253:36663,127.5.54.252:43033
16:30:52.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/krb5kdc/kudu_127.5.54.254.keytab
16:30:52.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.5.54.254
16:30:52.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:30:52.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:30:52.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:30:52.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.5.54.254:39175
16:30:52.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:30:52.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:30:52.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:30:52.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37491/jwks.json
16:30:52.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:30:52.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:30:52.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:30:52.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:30:52.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:30:52.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-0/data/info.pb
16:30:52.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.5.54.254
16:30:52.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36675
16:30:52.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:30:52.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:30:52.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:30:52.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:30:52.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:30:52.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-0/logs
16:30:52.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:30:52.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:30:52.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:30:52.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:30:52.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
16:30:52.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
16:30:52.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:30:52.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 15:11:41 UTC on ccycloud.quasar-yukscv.root.comops.site
16:30:52.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4674342
16:30:52.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:30:52.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:52.568713 5627 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:30:52.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:52.569903 5627 file_cache.cc:493] Constructed file cache file cache with capacity 419430
16:30:52.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:52.590595 5633 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:30:53.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:52.590583 5634 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:30:53.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:52.593734 5627 server_base.cc:1034] running on GCE node
16:30:53.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:52.590803 5636 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:30:53.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:53.887085 5627 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:30:53.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:53.889603 5627 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:30:53.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:53.891033 5627 hybrid_clock.cc:648] HybridClock initialized: now 1790440253890965 us; error 42 us; skew 500 ppm
16:30:53.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:53.891887 5627 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:30:53.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:30:53 dist-test-slave-lbns krb5kdc[5351](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790440253, etypes {rep=17 tkt=17 ses=17}, kudu/127.5.54.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:30:53.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:53.897660 5627 init.cc:377] Logged in from keytab as kudu/127.5.54.254@KRBTEST.COM (short username kudu)
16:30:53.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:53.903182 5627 webserver.cc:460] Webserver started at http://127.5.54.254:36675/ using document root <none> and password file <none>
16:30:53.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:53.904134 5627 fs_manager.cc:359] Metadata directory not provided
16:30:53.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:53.904388 5627 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:30:53.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:53.912681 5627 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.005s sys 0.000s
16:30:53.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:53.917052 5645 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:30:53.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:53.918109 5627 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
16:30:53.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:53.918483 5627 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-0/data,/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-0/wal
16:30:53.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f364d2622ec34443a4604cce6270022d"
16:30:53.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:30:47 on dist-test-slave-lbns"
16:30:53.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:53.921460 5627 fs_report.cc:383] FS layout report
16:30:53.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:30:53.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-0/wal
16:30:53.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-0/wal
16:30:53.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-0/data/data
16:30:53.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:30:53.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:30:53.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:30:53.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:30:53.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:30:53.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:30:53.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:30:53.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:30:53.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:30:53.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:30:53.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:30:53.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:53.967691 5627 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
16:30:53.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:53.973366 5627 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:30:53.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:53.973882 5627 kserver.cc:163] Server-wide thread pool size limit: 3276
16:30:54.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:54.044188 5627 rpc_server.cc:307] RPC server started. Bound to: 127.5.54.254:39175
16:30:54.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:54.044342 5696 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.5.54.254:39175 every 8 connection(s)
16:30:54.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:54.047658 5627 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-0/data/info.pb
16:30:54.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:54.054361 5339 external_mini_cluster.cc:1388] Started /tmp/dist-test-task4YrI62/build/tsan/bin/kudu as pid 5627
16:30:54.055 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:30:54.055 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.5.54.252:43033
16:30:54.055 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
16:30:54.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:54.057610 5339 external_mini_cluster.cc:1326] Running /tmp/dist-test-task4YrI62/build/tsan/bin/kudu
16:30:54.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4YrI62/build/tsan/bin/kudu
16:30:54.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-2/wal
16:30:54.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-2/data
16:30:54.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:30:54.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:30:54.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:30:54.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:30:54.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:30:54.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:30:54.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-2/logs
16:30:54.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-2/data/info.pb
16:30:54.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:30:54.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:30:54.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:30:54.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:30:54.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:30:54.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:30:54.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:30:54.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:30:54.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:30:54.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:30:54.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:30:54.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:30:54.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.5.54.252:43033
16:30:54.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.5.54.252
16:30:54.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37353
16:30:54.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/krb5kdc/kudu_127.5.54.252.keytab
16:30:54.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.5.54.252
16:30:54.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:30:54.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:30:54.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:30:54.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.5.54.212:38523
16:30:54.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:30:54.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:30:54.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:30:54.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:30:54.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.5.54.254:39175,127.5.54.253:36663,127.5.54.252:43033
16:30:54.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:30:54.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37491/jwks.json
16:30:54.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:30:54.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:54.064757 5697 sys_catalog.cc:263] Verifying existing consensus state
16:30:54.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260926 16:30:54.065652 5697 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
16:30:54.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:54.066032 5627 master.cc:537] Master@127.5.54.254:39175 shutting down...
16:30:54.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:54.128696 5184 debug-util.cc:398] Leaking SignalData structure 0x7b08000b5540 after lost signal to thread 5059
16:30:54.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:54.129845 5184 debug-util.cc:398] Leaking SignalData structure 0x7b08000b5260 after lost signal to thread 5187
16:30:54.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:54.134107 5627 master.cc:552] Master@127.5.54.254:39175 shutdown complete.
16:30:54.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
16:30:54.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:54.436860 5699 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:30:54.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:54.437623 5699 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:30:54.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:54.438669 5699 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:30:54.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:54.476975 5699 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
16:30:54.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:54.477283 5699 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:30:54.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:54.477684 5699 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:30:54.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:54.478026 5699 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:30:54.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:54.478354 5699 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:30:54.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:54.478716 5699 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:30:54.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:54.478955 5699 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37491/jwks.json
16:30:54.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:54.517362 5699 master_runner.cc:386] Master server non-default flags:
16:30:54.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:30:54.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.5.54.212:38523
16:30:54.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:30:54.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:30:54.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-2/data
16:30:54.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-2/wal
16:30:54.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
16:30:54.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:30:54.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.5.54.254:39175,127.5.54.253:36663,127.5.54.252:43033
16:30:54.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/krb5kdc/kudu_127.5.54.252.keytab
16:30:54.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.5.54.252
16:30:54.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:30:54.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:30:54.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:30:54.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.5.54.252:43033
16:30:54.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:30:54.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:30:54.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:30:54.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37491/jwks.json
16:30:54.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:30:54.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:30:54.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:30:54.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:30:54.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:30:54.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-2/data/info.pb
16:30:54.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.5.54.252
16:30:54.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37353
16:30:54.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:30:54.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:30:54.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:30:54.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:30:54.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:30:54.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-2/logs
16:30:54.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:30:54.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:30:54.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:30:54.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:30:54.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
16:30:54.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
16:30:54.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:30:54.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 15:11:41 UTC on ccycloud.quasar-yukscv.root.comops.site
16:30:54.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4674342
16:30:54.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:30:54.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:54.518934 5699 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:30:54.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:54.520500 5699 file_cache.cc:493] Constructed file cache file cache with capacity 419430
16:30:54.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:54.543220 5707 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:30:55.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:54.543629 5709 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:30:55.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:54.545106 5706 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:30:55.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:54.546644 5699 server_base.cc:1034] running on GCE node
16:30:55.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:55.913563 5699 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:30:55.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:55.918088 5699 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:30:55.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:55.919644 5699 hybrid_clock.cc:648] HybridClock initialized: now 1790440255919592 us; error 52 us; skew 500 ppm
16:30:55.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:55.920594 5699 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:30:55.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:30:55 dist-test-slave-lbns krb5kdc[5351](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790440255, etypes {rep=17 tkt=17 ses=17}, kudu/127.5.54.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:30:55.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:55.932881 5699 init.cc:377] Logged in from keytab as kudu/127.5.54.252@KRBTEST.COM (short username kudu)
16:30:55.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:55.939220 5699 webserver.cc:460] Webserver started at http://127.5.54.252:37353/ using document root <none> and password file <none>
16:30:55.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:55.940440 5699 fs_manager.cc:359] Metadata directory not provided
16:30:55.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:55.940661 5699 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:30:55.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:55.949443 5699 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.005s sys 0.000s
16:30:55.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:55.954077 5718 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:30:55.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:55.955634 5699 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
16:30:55.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:55.956084 5699 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-2/data,/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-2/wal
16:30:55.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7f2fd70ed2b0475184279df962be20a3"
16:30:55.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:30:51 on dist-test-slave-lbns"
16:30:55.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:55.958787 5699 fs_report.cc:383] FS layout report
16:30:55.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:30:55.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-2/wal
16:30:55.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-2/wal
16:30:55.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-2/data/data
16:30:55.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:30:55.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:30:55.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:30:55.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:30:55.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:30:55.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:30:55.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:30:55.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:30:55.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:30:55.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:30:55.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:30:56.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:56.006877 5699 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
16:30:56.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:56.013226 5699 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:30:56.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:56.013814 5699 kserver.cc:163] Server-wide thread pool size limit: 3276
16:30:56.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:56.090243 5699 rpc_server.cc:307] RPC server started. Bound to: 127.5.54.252:43033
16:30:56.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:56.090294 5769 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.5.54.252:43033 every 8 connection(s)
16:30:56.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:56.093771 5699 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-2/data/info.pb
16:30:56.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:56.103709 5339 external_mini_cluster.cc:1388] Started /tmp/dist-test-task4YrI62/build/tsan/bin/kudu as pid 5699
16:30:56.105 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:30:56.105 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.5.54.253:36663
16:30:56.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:56.105414 5770 sys_catalog.cc:263] Verifying existing consensus state
16:30:56.105 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
16:30:56.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:56.107367 5339 external_mini_cluster.cc:1326] Running /tmp/dist-test-task4YrI62/build/tsan/bin/kudu
16:30:56.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4YrI62/build/tsan/bin/kudu
16:30:56.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-1/wal
16:30:56.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-1/data
16:30:56.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:30:56.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:30:56.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:30:56.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:30:56.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:30:56.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:30:56.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-1/logs
16:30:56.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-1/data/info.pb
16:30:56.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:30:56.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:30:56.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:30:56.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:30:56.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:30:56.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:30:56.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:30:56.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:30:56.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:30:56.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:30:56.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:30:56.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:30:56.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.5.54.253:36663
16:30:56.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.5.54.253
16:30:56.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45717
16:30:56.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/krb5kdc/kudu_127.5.54.253.keytab
16:30:56.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.5.54.253
16:30:56.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:30:56.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:30:56.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:30:56.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.5.54.212:38523
16:30:56.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:30:56.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:30:56.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:30:56.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:30:56.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.5.54.254:39175,127.5.54.253:36663,127.5.54.252:43033
16:30:56.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:30:56.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37491/jwks.json
16:30:56.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:30:56.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:56.112890 5770 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3: Bootstrap starting.
16:30:56.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:56.157948 5770 log.cc:824] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3: Log is configured to *not* fsync() on all Append() calls
16:30:56.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:56.162930 5770 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
16:30:56.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:56.163929 5770 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3: Bootstrap complete.
16:30:56.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:56.188961 5770 raft_consensus.cc:348] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f364d2622ec34443a4604cce6270022d" member_type: VOTER last_known_addr { host: "127.5.54.254" port: 39175 } } peers { permanent_uuid: "e4f97fde47054172b47f501330f57c2a" member_type: VOTER last_known_addr { host: "127.5.54.253" port: 36663 } } peers { permanent_uuid: "7f2fd70ed2b0475184279df962be20a3" member_type: VOTER last_known_addr { host: "127.5.54.252" port: 43033 } }
16:30:56.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:56.191613 5770 raft_consensus.cc:374] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
16:30:56.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:56.191926 5770 raft_consensus.cc:729] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7f2fd70ed2b0475184279df962be20a3, State: Initialized, Role: FOLLOWER
16:30:56.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:56.192962 5770 consensus_queue.cc:260] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f364d2622ec34443a4604cce6270022d" member_type: VOTER last_known_addr { host: "127.5.54.254" port: 39175 } } peers { permanent_uuid: "e4f97fde47054172b47f501330f57c2a" member_type: VOTER last_known_addr { host: "127.5.54.253" port: 36663 } } peers { permanent_uuid: "7f2fd70ed2b0475184279df962be20a3" member_type: VOTER last_known_addr { host: "127.5.54.252" port: 43033 } }
16:30:56.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:56.196699 5774 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f364d2622ec34443a4604cce6270022d" member_type: VOTER last_known_addr { host: "127.5.54.254" port: 39175 } } peers { permanent_uuid: "e4f97fde47054172b47f501330f57c2a" member_type: VOTER last_known_addr { host: "127.5.54.253" port: 36663 } } peers { permanent_uuid: "7f2fd70ed2b0475184279df962be20a3" member_type: VOTER last_known_addr { host: "127.5.54.252" port: 43033 } } }
16:30:56.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:56.197861 5770 sys_catalog.cc:564] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3 [sys.catalog]: configured and running, proceeding with master startup.
16:30:56.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:56.197955 5774 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3 [sys.catalog]: This master's current role is: FOLLOWER
16:30:56.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:56.240703 5785 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:30:56.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:56.241056 5785 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:30:56.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:56.269997 5724 tablet.cc:2369] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3: Can't schedule compaction. Clean time has not been advanced past its initial value.
16:30:56.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:56.238381 5699 master_runner.cc:186] Error getting master registration for 127.5.54.254:39175: Network error: Client connection negotiation failed: client connection to 127.5.54.254:39175: connect: Connection refused (error 111),
16:30:56.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:56.276238 5699 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:30:56.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:56.500242 5772 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:30:56.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:56.500773 5772 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:30:56.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:56.501364 5772 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:30:56.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:56.533758 5772 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
16:30:56.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:56.534008 5772 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:30:56.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:56.534267 5772 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:30:56.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:56.534444 5772 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:30:56.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:56.534708 5772 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:30:56.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:56.534894 5772 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:30:56.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:56.535033 5772 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37491/jwks.json
16:30:56.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:56.546134 5774 raft_consensus.cc:482] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
16:30:56.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:56.546734 5774 raft_consensus.cc:504] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f364d2622ec34443a4604cce6270022d" member_type: VOTER last_known_addr { host: "127.5.54.254" port: 39175 } } peers { permanent_uuid: "e4f97fde47054172b47f501330f57c2a" member_type: VOTER last_known_addr { host: "127.5.54.253" port: 36663 } } peers { permanent_uuid: "7f2fd70ed2b0475184279df962be20a3" member_type: VOTER last_known_addr { host: "127.5.54.252" port: 43033 } }
16:30:56.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:56.550550 5774 leader_election.cc:288] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f364d2622ec34443a4604cce6270022d (127.5.54.254:39175), e4f97fde47054172b47f501330f57c2a (127.5.54.253:36663)
16:30:56.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:56.552225 5720 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.5.54.254:39175: connect: Connection refused (error 111)
16:30:56.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:56.563840 5720 leader_election.cc:334] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer f364d2622ec34443a4604cce6270022d (127.5.54.254:39175): Network error: Client connection negotiation failed: client connection to 127.5.54.254:39175: connect: Connection refused (error 111)
16:30:56.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:56.564718 5722 leader_election.cc:334] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e4f97fde47054172b47f501330f57c2a (127.5.54.253:36663): Network error: Client connection negotiation failed: client connection to 127.5.54.253:36663: connect: Connection refused (error 111)
16:30:56.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:56.565711 5722 leader_election.cc:302] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 7f2fd70ed2b0475184279df962be20a3; no voters: e4f97fde47054172b47f501330f57c2a, f364d2622ec34443a4604cce6270022d
16:30:56.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:56.566974 5774 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
16:30:56.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:56.583812 5772 master_runner.cc:386] Master server non-default flags:
16:30:56.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:30:56.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.5.54.212:38523
16:30:56.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:30:56.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:30:56.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-1/data
16:30:56.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-1/wal
16:30:56.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
16:30:56.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:30:56.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.5.54.254:39175,127.5.54.253:36663,127.5.54.252:43033
16:30:56.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/krb5kdc/kudu_127.5.54.253.keytab
16:30:56.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.5.54.253
16:30:56.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:30:56.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:30:56.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:30:56.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.5.54.253:36663
16:30:56.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:30:56.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:30:56.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:30:56.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37491/jwks.json
16:30:56.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:30:56.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:30:56.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:30:56.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:30:56.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:30:56.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-1/data/info.pb
16:30:56.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.5.54.253
16:30:56.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45717
16:30:56.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:30:56.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:30:56.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:30:56.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:30:56.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:30:56.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-1/logs
16:30:56.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:30:56.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:30:56.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:30:56.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:30:56.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
16:30:56.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
16:30:56.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:30:56.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 15:11:41 UTC on ccycloud.quasar-yukscv.root.comops.site
16:30:56.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4674342
16:30:56.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:30:56.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:56.584918 5772 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:30:56.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:56.585861 5772 file_cache.cc:493] Constructed file cache file cache with capacity 419430
16:30:56.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:56.608160 5795 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:30:57.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:57.245694 5785 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:30:57.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:57.246618 5785 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:30:58.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:58.251320 5785 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:30:58.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:58.251911 5785 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:30:58.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:58.002167 5794 debug-util.cc:398] Leaking SignalData structure 0x7b0800014920 after lost signal to thread 5772
16:30:58.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:58.420544 5794 kernel_stack_watchdog.cc:198] Thread 5772 stuck at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:637 for 391ms:
16:30:58.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
16:30:58.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
16:30:58.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
16:30:58.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
16:30:58.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:56.609694 5796 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:30:58.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:58.422760 5772 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 1.814s user 0.508s sys 0.962s
16:30:58.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:58.423271 5772 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 1.815s user 0.508s sys 0.962s
16:30:58.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:58.423796 5798 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:30:58.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:58.428282 5797 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1818 milliseconds
16:30:58.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:58.428336 5772 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:30:58.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:58.429850 5772 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:30:58.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:58.432618 5772 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:30:58.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:58.434043 5772 hybrid_clock.cc:648] HybridClock initialized: now 1790440258433978 us; error 86 us; skew 500 ppm
16:30:58.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:58.435014 5772 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:30:58.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:30:58 dist-test-slave-lbns krb5kdc[5351](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790440258, etypes {rep=17 tkt=17 ses=17}, kudu/127.5.54.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:30:58.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:58.441551 5772 init.cc:377] Logged in from keytab as kudu/127.5.54.253@KRBTEST.COM (short username kudu)
16:30:58.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:58.447553 5772 webserver.cc:460] Webserver started at http://127.5.54.253:45717/ using document root <none> and password file <none>
16:30:58.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:58.448601 5772 fs_manager.cc:359] Metadata directory not provided
16:30:58.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:58.448874 5772 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:30:58.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:58.460250 5772 fs_manager.cc:709] Time spent opening directory manager: real 0.007s user 0.006s sys 0.002s
16:30:58.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:58.467208 5806 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:30:58.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:58.468614 5772 fs_manager.cc:725] Time spent opening block manager: real 0.006s user 0.003s sys 0.001s
16:30:58.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:58.469059 5772 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-1/data,/tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-1/wal
16:30:58.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e4f97fde47054172b47f501330f57c2a"
16:30:58.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 16:30:49 on dist-test-slave-lbns"
16:30:58.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:58.471698 5772 fs_report.cc:383] FS layout report
16:30:58.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:30:58.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-1/wal
16:30:58.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-1/wal
16:30:58.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-1/data/data
16:30:58.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:30:58.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:30:58.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:30:58.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:30:58.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:30:58.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:30:58.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:30:58.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:30:58.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:30:58.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:30:58.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:30:58.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:58.517864 5772 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
16:30:58.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:58.525651 5772 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:30:58.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:58.526232 5772 kserver.cc:163] Server-wide thread pool size limit: 3276
16:30:58.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:58.614584 5772 rpc_server.cc:307] RPC server started. Bound to: 127.5.54.253:36663
16:30:58.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:58.614737 5857 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.5.54.253:36663 every 8 connection(s)
16:30:58.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:58.618048 5772 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-1/data/info.pb
16:30:58.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:58.621853 5339 external_mini_cluster.cc:1388] Started /tmp/dist-test-task4YrI62/build/tsan/bin/kudu as pid 5772
16:30:58.623 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:30:58.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:58.638794 5859 sys_catalog.cc:263] Verifying existing consensus state
16:30:58.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260926 16:30:58.639835 5859 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
16:30:58.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:58.640066 5860 raft_consensus.cc:482] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
16:30:58.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:58.640261 5772 master.cc:537] Master@127.5.54.253:36663 shutting down...
16:30:58.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:58.644016 5860 raft_consensus.cc:504] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f364d2622ec34443a4604cce6270022d" member_type: VOTER last_known_addr { host: "127.5.54.254" port: 39175 } } peers { permanent_uuid: "e4f97fde47054172b47f501330f57c2a" member_type: VOTER last_known_addr { host: "127.5.54.253" port: 36663 } } peers { permanent_uuid: "7f2fd70ed2b0475184279df962be20a3" member_type: VOTER last_known_addr { host: "127.5.54.252" port: 43033 } }
16:30:58.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:58.646418 5860 leader_election.cc:288] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f364d2622ec34443a4604cce6270022d (127.5.54.254:39175), e4f97fde47054172b47f501330f57c2a (127.5.54.253:36663)
16:30:58.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:58.684046 5720 leader_election.cc:334] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer f364d2622ec34443a4604cce6270022d (127.5.54.254:39175): Network error: Client connection negotiation failed: client connection to 127.5.54.254:39175: connect: Connection refused (error 111)
16:30:58.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:30:58 dist-test-slave-lbns krb5kdc[5351](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790440255, etypes {rep=17 tkt=17 ses=17}, kudu/127.5.54.252@KRBTEST.COM for kudu/127.5.54.253@KRBTEST.COM
16:30:58.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:58.736980 5722 connection.cc:535] client connection to 127.5.54.253:36663 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
16:30:58.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:58.737432 5722 leader_election.cc:334] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e4f97fde47054172b47f501330f57c2a (127.5.54.253:36663): Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
16:30:58.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:58.737834 5722 leader_election.cc:302] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 7f2fd70ed2b0475184279df962be20a3; no voters: e4f97fde47054172b47f501330f57c2a, f364d2622ec34443a4604cce6270022d
16:30:58.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:58.738760 5860 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
16:30:58.740 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.5.54.254:39175(127.5.54.254:39175)] connecting to peer
16:30:58.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:30:58.806805 5772 master.cc:552] Master@127.5.54.253:36663 shutdown complete.
16:30:58.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task4YrI62/test-tmp/mini-kudu-cluster8501789715946526569/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
16:30:58.834 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.5.54.252:43033(127.5.54.252:43033)] connecting to peer
16:30:58.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:58.837282 5044 debug-util.cc:398] Leaking SignalData structure 0x7b08000af740 after lost signal to thread 4915
16:30:58.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:58.838380 5044 debug-util.cc:398] Leaking SignalData structure 0x7b08000be0c0 after lost signal to thread 5047
16:30:58.839 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.5.54.253:36663(127.5.54.253:36663)] connecting to peer
16:30:58.899 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.5.54.252:43033(127.5.54.252:43033)] Successfully connected to peer
16:30:58.901 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:30:58.904 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.54.254:39175(127.5.54.254:39175)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:30:58.911 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.54.254:39175: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:30:58.913 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:30:58.913 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.54.253:36663(127.5.54.253:36663)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:30:58.920 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.54.253:36663: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:30:59.013 [WARN - kudu-nio-1] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
16:30:59.025 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.5.54.252
16:30:59.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:59.256506 5785 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:30:59.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:30:59.256855 5785 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:30:59.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:30:59 dist-test-slave-lbns krb5kdc[5351](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790440252, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.5.54.252@KRBTEST.COM
16:30:59.380 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xc9f024bd, L:/127.0.0.1:57174 - R:/127.5.54.252:43033] using SASL/GSSAPI
16:30:59.399 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663; will retry
16:30:59.420 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.54.254:39175(127.5.54.254:39175)] connecting to peer
16:30:59.424 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.54.253:36663(127.5.54.253:36663)] connecting to peer
16:30:59.427 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:30:59.427 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.54.253:36663(127.5.54.253:36663)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:30:59.428 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.54.253:36663: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:30:59.430 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:30:59.431 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.54.254:39175(127.5.54.254:39175)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:30:59.431 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.54.254:39175: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:30:59.434 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663; will retry
16:30:59.439 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.54.254:39175(127.5.54.254:39175)] connecting to peer
16:30:59.441 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.54.253:36663(127.5.54.253:36663)] connecting to peer
16:30:59.444 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:30:59.444 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.54.253:36663(127.5.54.253:36663)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:30:59.444 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.54.253:36663: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:30:59.446 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:30:59.446 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.54.254:39175(127.5.54.254:39175)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:30:59.447 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.54.254:39175: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:30:59.450 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663; will retry
16:30:59.459 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.54.254:39175(127.5.54.254:39175)] connecting to peer
16:30:59.461 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.54.253:36663(127.5.54.253:36663)] connecting to peer
16:30:59.465 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:30:59.465 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.54.253:36663(127.5.54.253:36663)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:30:59.467 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:30:59.467 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.54.253:36663: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:30:59.467 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.54.254:39175(127.5.54.254:39175)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:30:59.468 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.54.254:39175: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:30:59.468 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663; will retry
16:30:59.479 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.54.254:39175(127.5.54.254:39175)] connecting to peer
16:30:59.481 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.54.253:36663(127.5.54.253:36663)] connecting to peer
16:30:59.483 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:30:59.483 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.54.253:36663(127.5.54.253:36663)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:30:59.484 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.54.253:36663: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:30:59.485 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:30:59.485 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.54.254:39175(127.5.54.254:39175)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:30:59.486 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.54.254:39175: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:30:59.488 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663; will retry
16:30:59.519 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.54.254:39175(127.5.54.254:39175)] connecting to peer
16:30:59.521 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.54.253:36663(127.5.54.253:36663)] connecting to peer
16:30:59.522 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:30:59.523 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.54.253:36663(127.5.54.253:36663)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:30:59.526 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.54.253:36663: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:30:59.527 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:30:59.528 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.54.254:39175(127.5.54.254:39175)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:30:59.528 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.54.254:39175: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:30:59.529 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663; will retry
16:30:59.559 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.54.254:39175(127.5.54.254:39175)] connecting to peer
16:30:59.561 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.54.253:36663(127.5.54.253:36663)] connecting to peer
16:30:59.563 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:30:59.563 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.54.253:36663(127.5.54.253:36663)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:30:59.564 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.54.253:36663: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:30:59.568 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:30:59.569 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.54.254:39175(127.5.54.254:39175)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:30:59.569 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.54.254:39175: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:30:59.570 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663; will retry
16:30:59.619 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.54.254:39175(127.5.54.254:39175)] connecting to peer
16:30:59.621 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.54.253:36663(127.5.54.253:36663)] connecting to peer
16:30:59.624 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:30:59.624 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.54.253:36663(127.5.54.253:36663)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:30:59.624 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:30:59.624 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.54.254:39175(127.5.54.254:39175)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:30:59.626 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.54.253:36663: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:30:59.626 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.54.254:39175: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:30:59.628 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663; will retry
16:30:59.719 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.54.254:39175(127.5.54.254:39175)] connecting to peer
16:30:59.722 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.54.253:36663(127.5.54.253:36663)] connecting to peer
16:30:59.724 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:30:59.724 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.54.253:36663(127.5.54.253:36663)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:30:59.725 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.54.253:36663: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:30:59.725 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:30:59.726 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.54.254:39175(127.5.54.254:39175)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:30:59.726 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.54.254:39175: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:30:59.728 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663; will retry
16:30:59.898 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.54.254:39175(127.5.54.254:39175)] connecting to peer
16:30:59.900 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.54.253:36663(127.5.54.253:36663)] connecting to peer
16:30:59.907 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:30:59.907 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.54.253:36663(127.5.54.253:36663)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:30:59.908 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.54.253:36663: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:30:59.909 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:30:59.909 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.54.254:39175(127.5.54.254:39175)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:30:59.909 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.54.254:39175: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:30:59.910 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663; will retry
16:31:00.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:00.259925 5785 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:31:00.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:00.260229 5785 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:31:00.659 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.54.254:39175(127.5.54.254:39175)] connecting to peer
16:31:00.664 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:31:00.664 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.54.254:39175(127.5.54.254:39175)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:31:00.664 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.54.254:39175: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:31:00.668 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.54.253:36663(127.5.54.253:36663)] connecting to peer
16:31:00.674 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:31:00.674 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.54.253:36663(127.5.54.253:36663)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:31:00.674 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.54.253:36663: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:31:00.675 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663; will retry
16:31:00.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:31:00.686084 5872 raft_consensus.cc:482] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
16:31:00.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:31:00.686614 5872 raft_consensus.cc:504] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f364d2622ec34443a4604cce6270022d" member_type: VOTER last_known_addr { host: "127.5.54.254" port: 39175 } } peers { permanent_uuid: "e4f97fde47054172b47f501330f57c2a" member_type: VOTER last_known_addr { host: "127.5.54.253" port: 36663 } } peers { permanent_uuid: "7f2fd70ed2b0475184279df962be20a3" member_type: VOTER last_known_addr { host: "127.5.54.252" port: 43033 } }
16:31:00.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:31:00.688717 5872 leader_election.cc:288] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f364d2622ec34443a4604cce6270022d (127.5.54.254:39175), e4f97fde47054172b47f501330f57c2a (127.5.54.253:36663)
16:31:00.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:00.700945 5720 leader_election.cc:334] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer f364d2622ec34443a4604cce6270022d (127.5.54.254:39175): Network error: Client connection negotiation failed: client connection to 127.5.54.254:39175: connect: Connection refused (error 111)
16:31:00.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:00.702652 5722 leader_election.cc:334] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e4f97fde47054172b47f501330f57c2a (127.5.54.253:36663): Network error: Client connection negotiation failed: client connection to 127.5.54.253:36663: connect: Connection refused (error 111)
16:31:00.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:31:00.704015 5722 leader_election.cc:302] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 7f2fd70ed2b0475184279df962be20a3; no voters: e4f97fde47054172b47f501330f57c2a, f364d2622ec34443a4604cce6270022d
16:31:00.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:31:00.705219 5872 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
16:31:01.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:01.263271 5785 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:31:01.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:01.263572 5785 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:31:01.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:31:01.278967 5699 master_runner.cc:186] Error getting master registration for 127.5.54.254:39175: Network error: Client connection negotiation failed: client connection to 127.5.54.254:39175: connect: Connection refused (error 111),
16:31:01.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:31:01.279345 5699 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:31:01.438 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.54.254:39175(127.5.54.254:39175)] connecting to peer
16:31:01.440 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.54.253:36663(127.5.54.253:36663)] connecting to peer
16:31:01.442 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:31:01.443 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.54.254:39175(127.5.54.254:39175)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:31:01.445 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.54.254:39175: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:31:01.445 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:31:01.445 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.54.253:36663(127.5.54.253:36663)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:31:01.445 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.54.253:36663: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:31:01.446 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663; will retry
16:31:02.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:02.266459 5785 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:31:02.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:02.266810 5785 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:31:03.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:03.269767 5785 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:31:03.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:03.270036 5785 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:31:03.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:03.430974 5324 debug-util.cc:398] Leaking SignalData structure 0x7b08000ba040 after lost signal to thread 5199
16:31:03.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:03.431849 5324 debug-util.cc:398] Leaking SignalData structure 0x7b08000afc80 after lost signal to thread 5327
16:31:03.818 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.54.254:39175(127.5.54.254:39175)] connecting to peer
16:31:03.820 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:31:03.821 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.54.254:39175(127.5.54.254:39175)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:31:03.821 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.54.254:39175: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:31:03.821 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.54.253:36663(127.5.54.253:36663)] connecting to peer
16:31:03.823 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:31:03.824 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.54.253:36663(127.5.54.253:36663)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:31:03.824 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.54.253:36663: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:31:03.825 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663; will retry
16:31:04.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:04.274437 5785 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:31:04.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:04.274837 5785 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:31:04.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:04.975193 5766 debug-util.cc:398] Leaking SignalData structure 0x7b08000bd420 after lost signal to thread 5700
16:31:04.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:04.976455 5766 debug-util.cc:398] Leaking SignalData structure 0x7b08000bc4a0 after lost signal to thread 5769
16:31:05.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:05.279210 5785 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:31:05.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:05.279588 5785 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:31:05.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:31:05.678370 5884 raft_consensus.cc:482] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
16:31:05.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:31:05.678996 5884 raft_consensus.cc:504] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f364d2622ec34443a4604cce6270022d" member_type: VOTER last_known_addr { host: "127.5.54.254" port: 39175 } } peers { permanent_uuid: "e4f97fde47054172b47f501330f57c2a" member_type: VOTER last_known_addr { host: "127.5.54.253" port: 36663 } } peers { permanent_uuid: "7f2fd70ed2b0475184279df962be20a3" member_type: VOTER last_known_addr { host: "127.5.54.252" port: 43033 } }
16:31:05.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:31:05.681216 5884 leader_election.cc:288] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f364d2622ec34443a4604cce6270022d (127.5.54.254:39175), e4f97fde47054172b47f501330f57c2a (127.5.54.253:36663)
16:31:05.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:05.683940 5720 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.5.54.254:39175: connect: Connection refused (error 111) [suppressed 5 similar messages]
16:31:05.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:05.687228 5722 leader_election.cc:334] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e4f97fde47054172b47f501330f57c2a (127.5.54.253:36663): Network error: Client connection negotiation failed: client connection to 127.5.54.253:36663: connect: Connection refused (error 111)
16:31:05.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:05.688103 5720 leader_election.cc:334] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer f364d2622ec34443a4604cce6270022d (127.5.54.254:39175): Network error: Client connection negotiation failed: client connection to 127.5.54.254:39175: connect: Connection refused (error 111)
16:31:05.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:31:05.688516 5720 leader_election.cc:302] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 7f2fd70ed2b0475184279df962be20a3; no voters: e4f97fde47054172b47f501330f57c2a, f364d2622ec34443a4604cce6270022d
16:31:05.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:31:05.689342 5884 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
16:31:05.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:31:05 dist-test-slave-lbns krb5kdc[5351](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790440265, etypes {rep=17 tkt=17 ses=17}, kudu/127.5.54.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:31:05.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:31:05.938099 5714 init.cc:295] Successfully reacquired a new kerberos TGT
16:31:06.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:06.282902 5785 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:31:06.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:06.283154 5785 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:31:06.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:31:06.316355 5699 master_runner.cc:186] Error getting master registration for 127.5.54.254:39175: Network error: Client connection negotiation failed: client connection to 127.5.54.254:39175: connect: Connection refused (error 111),
16:31:06.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:31:06.316815 5699 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:31:07.018 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.54.254:39175(127.5.54.254:39175)] connecting to peer
16:31:07.021 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.54.253:36663(127.5.54.253:36663)] connecting to peer
16:31:07.022 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:31:07.022 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:31:07.022 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.54.253:36663(127.5.54.253:36663)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:31:07.022 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.54.254:39175(127.5.54.254:39175)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:31:07.023 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.54.254:39175: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:31:07.023 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.54.253:36663: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:31:07.024 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663; will retry
16:31:07.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:07.285887 5785 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:31:07.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:07.286140 5785 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:31:08.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:08.289395 5785 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:31:08.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:08.289666 5785 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:31:08.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:31:08.656644 5891 raft_consensus.cc:482] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
16:31:08.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:31:08.657063 5891 raft_consensus.cc:504] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f364d2622ec34443a4604cce6270022d" member_type: VOTER last_known_addr { host: "127.5.54.254" port: 39175 } } peers { permanent_uuid: "e4f97fde47054172b47f501330f57c2a" member_type: VOTER last_known_addr { host: "127.5.54.253" port: 36663 } } peers { permanent_uuid: "7f2fd70ed2b0475184279df962be20a3" member_type: VOTER last_known_addr { host: "127.5.54.252" port: 43033 } }
16:31:08.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:31:08.658771 5891 leader_election.cc:288] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f364d2622ec34443a4604cce6270022d (127.5.54.254:39175), e4f97fde47054172b47f501330f57c2a (127.5.54.253:36663)
16:31:08.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:08.663434 5722 leader_election.cc:334] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e4f97fde47054172b47f501330f57c2a (127.5.54.253:36663): Network error: Client connection negotiation failed: client connection to 127.5.54.253:36663: connect: Connection refused (error 111)
16:31:08.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:08.663847 5720 leader_election.cc:334] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer f364d2622ec34443a4604cce6270022d (127.5.54.254:39175): Network error: Client connection negotiation failed: client connection to 127.5.54.254:39175: connect: Connection refused (error 111)
16:31:08.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:31:08.664217 5720 leader_election.cc:302] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 7f2fd70ed2b0475184279df962be20a3; no voters: e4f97fde47054172b47f501330f57c2a, f364d2622ec34443a4604cce6270022d
16:31:08.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:31:08.664757 5891 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
16:31:08.878 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.54.254:39175(127.5.54.254:39175)] connecting to peer
16:31:08.880 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.54.253:36663(127.5.54.253:36663)] connecting to peer
16:31:08.882 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:31:08.882 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.54.253:36663(127.5.54.253:36663)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:31:08.883 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.54.253:36663: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:31:08.883 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:31:08.884 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.54.254:39175(127.5.54.254:39175)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:31:08.884 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.54.254:39175: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:31:08.885 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663; will retry
16:31:09.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:09.292593 5785 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:31:09.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:09.292902 5785 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:31:10.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:10.295801 5785 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:31:10.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:10.296061 5785 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:31:11.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:11.298936 5785 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:31:11.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:11.299197 5785 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:31:11.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:31:11.319519 5699 master_runner.cc:186] Error getting master registration for 127.5.54.254:39175: Network error: Client connection negotiation failed: client connection to 127.5.54.254:39175: connect: Connection refused (error 111),
16:31:11.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:31:11.319929 5699 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:31:12.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:12.301854 5785 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:31:12.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:12.302098 5785 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:31:12.578 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.54.254:39175(127.5.54.254:39175)] connecting to peer
16:31:12.580 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.54.253:36663(127.5.54.253:36663)] connecting to peer
16:31:12.582 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:31:12.582 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:31:12.582 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.54.253:36663(127.5.54.253:36663)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:31:12.582 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.54.254:39175(127.5.54.254:39175)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:31:12.582 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.54.254:39175: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:31:12.583 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.54.253:36663: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:31:12.584 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663; will retry
16:31:13.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:13.304803 5785 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:31:13.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:13.305056 5785 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:31:14.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:14.307842 5785 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:31:14.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:14.308151 5785 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:31:14.599 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.54.254:39175(127.5.54.254:39175)] connecting to peer
16:31:14.600 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.54.253:36663(127.5.54.253:36663)] connecting to peer
16:31:14.602 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:31:14.602 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.54.254:39175(127.5.54.254:39175)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:31:14.602 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:31:14.602 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.54.254:39175: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:31:14.602 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.54.253:36663(127.5.54.253:36663)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:31:14.602 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.54.253:36663: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:31:14.605 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663; will retry
16:31:15.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:15.311558 5785 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:31:15.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:15.311952 5785 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:31:15.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:31:15 dist-test-slave-lbns krb5kdc[5351](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790440275, etypes {rep=17 tkt=17 ses=17}, kudu/127.5.54.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:31:15.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:31:15.940747 5714 init.cc:295] Successfully reacquired a new kerberos TGT
16:31:16.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:31:16.051810 5906 raft_consensus.cc:482] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
16:31:16.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:31:16.052371 5906 raft_consensus.cc:504] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f364d2622ec34443a4604cce6270022d" member_type: VOTER last_known_addr { host: "127.5.54.254" port: 39175 } } peers { permanent_uuid: "e4f97fde47054172b47f501330f57c2a" member_type: VOTER last_known_addr { host: "127.5.54.253" port: 36663 } } peers { permanent_uuid: "7f2fd70ed2b0475184279df962be20a3" member_type: VOTER last_known_addr { host: "127.5.54.252" port: 43033 } }
16:31:16.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:31:16.054957 5906 leader_election.cc:288] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f364d2622ec34443a4604cce6270022d (127.5.54.254:39175), e4f97fde47054172b47f501330f57c2a (127.5.54.253:36663)
16:31:16.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:16.057614 5722 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.5.54.253:36663: connect: Connection refused (error 111) [suppressed 3 similar messages]
16:31:16.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:16.061127 5722 leader_election.cc:334] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e4f97fde47054172b47f501330f57c2a (127.5.54.253:36663): Network error: Client connection negotiation failed: client connection to 127.5.54.253:36663: connect: Connection refused (error 111)
16:31:16.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:16.071316 5720 leader_election.cc:334] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer f364d2622ec34443a4604cce6270022d (127.5.54.254:39175): Network error: Client connection negotiation failed: client connection to 127.5.54.254:39175: connect: Connection refused (error 111)
16:31:16.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:31:16.071659 5720 leader_election.cc:302] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 7f2fd70ed2b0475184279df962be20a3; no voters: e4f97fde47054172b47f501330f57c2a, f364d2622ec34443a4604cce6270022d
16:31:16.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:31:16.072345 5906 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
16:31:16.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:16.314745 5785 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:31:16.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:16.315083 5785 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:31:16.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:31:16.322144 5699 master_runner.cc:186] Error getting master registration for 127.5.54.254:39175: Network error: Client connection negotiation failed: client connection to 127.5.54.254:39175: connect: Connection refused (error 111),
16:31:16.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:31:16.322502 5699 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:31:17.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:17.318073 5785 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:31:17.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:17.318399 5785 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:31:17.698 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.54.254:39175(127.5.54.254:39175)] connecting to peer
16:31:17.700 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.54.253:36663(127.5.54.253:36663)] connecting to peer
16:31:17.702 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:31:17.702 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.54.253:36663(127.5.54.253:36663)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:31:17.702 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.54.253:36663: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:31:17.702 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:31:17.703 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.54.254:39175(127.5.54.254:39175)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:31:17.703 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.54.254:39175: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:31:17.704 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663; will retry
16:31:18.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:18.321938 5785 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:31:18.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:18.322222 5785 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:31:19.119 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.54.254:39175(127.5.54.254:39175)] connecting to peer
16:31:19.121 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.54.253:36663(127.5.54.253:36663)] connecting to peer
16:31:19.123 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:31:19.123 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:31:19.123 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.54.254:39175(127.5.54.254:39175)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:31:19.123 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.54.254:39175: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:31:19.123 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.54.253:36663(127.5.54.253:36663)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:31:19.124 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.54.253:36663: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:31:19.127 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663; will retry
16:31:19.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:19.324899 5785 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:31:19.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:19.325140 5785 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:31:20.078 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.54.254:39175(127.5.54.254:39175)] connecting to peer
16:31:20.080 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.54.253:36663(127.5.54.253:36663)] connecting to peer
16:31:20.082 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:31:20.082 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:31:20.082 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.54.254:39175(127.5.54.254:39175)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:31:20.082 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.54.253:36663(127.5.54.253:36663)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:31:20.082 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.54.254:39175: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:31:20.082 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.54.253:36663: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:31:20.084 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663; will retry
16:31:20.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:20.328027 5785 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:31:20.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:20.328276 5785 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:31:21.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:31:21.325230 5699 master_runner.cc:186] Error getting master registration for 127.5.54.254:39175: Network error: Client connection negotiation failed: client connection to 127.5.54.254:39175: connect: Connection refused (error 111),
16:31:21.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:31:21.325537 5699 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:31:21.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:21.331310 5785 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:31:21.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:21.331593 5785 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:31:22.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:22.334231 5785 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:31:22.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:22.334486 5785 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:31:22.919 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.54.254:39175(127.5.54.254:39175)] connecting to peer
16:31:22.920 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.54.253:36663(127.5.54.253:36663)] connecting to peer
16:31:22.921 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:31:22.921 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.54.253:36663(127.5.54.253:36663)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:31:22.922 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.54.253:36663: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:31:22.921 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:31:22.922 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.54.254:39175(127.5.54.254:39175)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:31:22.922 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.54.254:39175: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:31:22.923 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663; will retry
16:31:23.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:23.337284 5785 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:31:23.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:23.337612 5785 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:31:24.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:24.340809 5785 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:31:24.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:24.341082 5785 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:31:25.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:25.244171 4846 debug-util.cc:398] Leaking SignalData structure 0x7b080009f1e0 after lost signal to thread 4777
16:31:25.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:25.244843 4846 debug-util.cc:398] Leaking SignalData structure 0x7b08000bf180 after lost signal to thread 4849
16:31:25.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:25.344753 5785 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:31:25.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:25.345114 5785 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:31:25.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 16:31:25 dist-test-slave-lbns krb5kdc[5351](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790440285, etypes {rep=17 tkt=17 ses=17}, kudu/127.5.54.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:31:25.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:31:25.943873 5714 init.cc:295] Successfully reacquired a new kerberos TGT
16:31:26.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:31:26.329795 5699 master_runner.cc:186] Error getting master registration for 127.5.54.254:39175: Network error: Client connection negotiation failed: client connection to 127.5.54.254:39175: connect: Connection refused (error 111),
16:31:26.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:31:26.330349 5699 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:31:26.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:26.348904 5785 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:31:26.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:26.349197 5785 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:31:26.659 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.54.254:39175(127.5.54.254:39175)] connecting to peer
16:31:26.660 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.54.253:36663(127.5.54.253:36663)] connecting to peer
16:31:26.661 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:31:26.661 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.54.254:39175(127.5.54.254:39175)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:31:26.661 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:31:26.661 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.54.253:36663(127.5.54.253:36663)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:31:26.661 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.54.254:39175: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:31:26.661 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.54.253:36663: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:31:26.664 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663; will retry
16:31:27.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:31:27.172906 5922 raft_consensus.cc:482] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
16:31:27.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:31:27.173385 5922 raft_consensus.cc:504] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f364d2622ec34443a4604cce6270022d" member_type: VOTER last_known_addr { host: "127.5.54.254" port: 39175 } } peers { permanent_uuid: "e4f97fde47054172b47f501330f57c2a" member_type: VOTER last_known_addr { host: "127.5.54.253" port: 36663 } } peers { permanent_uuid: "7f2fd70ed2b0475184279df962be20a3" member_type: VOTER last_known_addr { host: "127.5.54.252" port: 43033 } }
16:31:27.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:31:27.175269 5922 leader_election.cc:288] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f364d2622ec34443a4604cce6270022d (127.5.54.254:39175), e4f97fde47054172b47f501330f57c2a (127.5.54.253:36663)
16:31:27.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:27.177225 5720 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.5.54.254:39175: connect: Connection refused (error 111) [suppressed 1 similar messages]
16:31:27.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:27.180262 5720 leader_election.cc:334] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer f364d2622ec34443a4604cce6270022d (127.5.54.254:39175): Network error: Client connection negotiation failed: client connection to 127.5.54.254:39175: connect: Connection refused (error 111)
16:31:27.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:27.180881 5722 leader_election.cc:334] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e4f97fde47054172b47f501330f57c2a (127.5.54.253:36663): Network error: Client connection negotiation failed: client connection to 127.5.54.253:36663: connect: Connection refused (error 111)
16:31:27.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:31:27.181250 5722 leader_election.cc:302] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 7f2fd70ed2b0475184279df962be20a3; no voters: e4f97fde47054172b47f501330f57c2a, f364d2622ec34443a4604cce6270022d
16:31:27.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:31:27.181975 5922 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
16:31:27.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:27.351634 5785 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7f2fd70ed2b0475184279df962be20a3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:31:27.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 16:31:27.351862 5785 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:31:27.699 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.54.254:39175(127.5.54.254:39175)] connecting to peer
16:31:27.699 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.54.253:36663(127.5.54.253:36663)] connecting to peer
16:31:27.700 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:31:27.700 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:31:27.701 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.54.254:39175(127.5.54.254:39175)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:31:27.701 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.54.253:36663(127.5.54.253:36663)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:31:27.701 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.54.254:39175: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175
16:31:27.701 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.54.253:36663: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
16:31:27.703 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663; will retry
16:31:27.707 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=23, TimeoutTracker(timeout=30000, elapsed=29075), Traces: [0ms] refreshing cache from master, [52ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.254:39175, [163ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.252:43033, [168ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.253:36663, [234ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.254:39175: Network error: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [242ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.253:36663: Network error: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [724ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.252:43033: OK, [729ms] delaying RPC due to: Service unavailable: Master config (127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [748ms] refreshing cache from master, [749ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.254:39175, [751ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.252:43033, [753ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.253:36663, [757ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.253:36663: Network error: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [760ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.254:39175: Network error: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [762ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.252:43033: OK, [763ms] delaying RPC due to: Service unavailable: Master config (127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [767ms] refreshing cache from master, [768ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.254:39175, [769ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.252:43033, [770ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.253:36663, [773ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.253:36663: Network error: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [775ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.254:39175: Network error: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [778ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.252:43033: OK, [779ms] delaying RPC due to: Service unavailable: Master config (127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [787ms] refreshing cache from master, [788ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.254:39175, [789ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.252:43033, [790ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.253:36663, [795ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.252:43033: OK, [795ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.253:36663: Network error: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [796ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.254:39175: Network error: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [798ms] delaying RPC due to: Service unavailable: Master config (127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [807ms] refreshing cache from master, [808ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.254:39175, [809ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.252:43033, [810ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.253:36663, [812ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.253:36663: Network error: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [814ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.254:39175: Network error: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [816ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.252:43033: OK, [817ms] delaying RPC due to: Service unavailable: Master config (127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [847ms] refreshing cache from master, [847ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.254:39175, [849ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.252:43033, [849ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.253:36663, [852ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.253:36663: Network error: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [855ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.252:43033: OK, [857ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.254:39175: Network error: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [858ms] delaying RPC due to: Service unavailable: Master config (127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [887ms] refreshing cache from master, [888ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.254:39175, [889ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.252:43033, [890ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.253:36663, [892ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.253:36663: Network error: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [897ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.252:43033: OK, [898ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.254:39175: Network error: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [899ms] delaying RPC due to: Service unavailable: Master config (127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [947ms] refreshing cache from master, [948ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.254:39175, [949ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.252:43033, [950ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.253:36663, [953ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.253:36663: Network error: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [954ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.254:39175: Network error: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [956ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.252:43033: OK, [957ms] delaying RPC due to: Service unavailable: Master config (127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [1047ms] refreshing cache from master, [1048ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.254:39175, [1050ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.252:43033, [1050ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.253:36663, [1053ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.253:36663: Network error: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [1055ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.254:39175: Network error: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [1057ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.252:43033: OK, [1058ms] delaying RPC due to: Service unavailable: Master config (127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [1227ms] refreshing cache from master, [1227ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.254:39175, [1228ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.252:43033, [1229ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.253:36663, [1236ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.253:36663: Network error: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [1237ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.252:43033: OK, [1238ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.254:39175: Network error: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [1239ms] delaying RPC due to: Service unavailable: Master config (127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [1988ms] refreshing cache from master, [1988ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.254:39175, [1993ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.252:43033, [1993ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.254:39175: Network error: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [1997ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.253:36663, [1999ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.252:43033: OK, [2003ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.253:36663: Network error: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [2004ms] delaying RPC due to: Service unavailable: Master config (127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [2767ms] refreshing cache from master, [2767ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.254:39175, [2768ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.252:43033, [2769ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.253:36663, [2773ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.254:39175: Network error: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [2774ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.253:36663: Network error: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [2775ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.252:43033: OK, [2775ms] delaying RPC due to: Service unavailable: Master config (127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [5147ms] refreshing cache from master, [5147ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.254:39175, [5149ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.252:43033, [5150ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.254:39175: Network error: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [5150ms] too many traces: truncated at 100 traces, deferred=Deferred@1706261723(state=PENDING, result=null, callback=(continuation of Deferred@680635496 after retry RPC after error@593135639) -> (continuation of Deferred@2085874 after retry RPC after error@2117345934) -> (continuation of Deferred@1541616588 after retry RPC after error@739119767) -> (continuation of Deferred@1383122087 after retry RPC after error@227523637) -> (continuation of Deferred@1336967963 after retry RPC after error@334932688) -> (continuation of Deferred@1198981621 after retry RPC after error@938042170) -> (continuation of Deferred@2081068679 after retry RPC after error@1510740988) -> (continuation of Deferred@621533665 after retry RPC after error@1502056137) -> (continuation of Deferred@1896973776 after retry RPC after error@1262025725) -> (continuation of Deferred@805824426 after retry RPC after error@1427359397) -> (continuation of Deferred@676460242 after retry RPC after error@607782710) -> (continuation of Deferred@730832556 after retry RPC after error@1617056662) -> (continuation of Deferred@1193463342 after retry RPC after error@861237835) -> (continuation of Deferred@1685647475 after retry RPC after error@376758665) -> (continuation of Deferred@1198923479 after retry RPC after error@1596582744) -> (continuation of Deferred@2087809958 after retry RPC after error@1295564040) -> (continuation of Deferred@986677982 after retry RPC after error@715967988) -> (continuation of Deferred@1563916100 after retry RPC after error@1441743127) -> (continuation of Deferred@334557190 after retry RPC after error@1610570977) -> (continuation of Deferred@957652237 after retry RPC after error@375166145) -> (continuation of Deferred@45622988 after retry RPC after error@663058990) -> (continuation of Deferred@979658534 after retry RPC after error@1215043127), errback=(continuation of Deferred@680635496 after retry RPC after error@593135639) -> (continuation of Deferred@2085874 after retry RPC after error@2117345934) -> (continuation of Deferred@1541616588 after retry RPC after error@739119767) -> (continuation of Deferred@1383122087 after retry RPC after error@227523637) -> (continuation of Deferred@1336967963 after retry RPC after error@334932688) -> (continuation of Deferred@1198981621 after retry RPC after error@938042170) -> (continuation of Deferred@2081068679 after retry RPC after error@1510740988) -> (continuation of Deferred@621533665 after retry RPC after error@1502056137) -> (continuation of Deferred@1896973776 after retry RPC after error@1262025725) -> (continuation of Deferred@805824426 after retry RPC after error@1427359397) -> (continuation of Deferred@676460242 after retry RPC after error@607782710) -> (continuation of Deferred@730832556 after retry RPC after error@1617056662) -> (continuation of Deferred@1193463342 after retry RPC after error@861237835) -> (continuation of Deferred@1685647475 after retry RPC after error@376758665) -> (continuation of Deferred@1198923479 after retry RPC after error@1596582744) -> (continuation of Deferred@2087809958 after retry RPC after error@1295564040) -> (continuation of Deferred@986677982 after retry RPC after error@715967988) -> (continuation of Deferred@1563916100 after retry RPC after error@1441743127) -> (continuation of Deferred@334557190 after retry RPC after error@1610570977) -> (continuation of Deferred@957652237 after retry RPC after error@375166145) -> (continuation of Deferred@45622988 after retry RPC after error@663058990) -> (continuation of Deferred@979658534 after retry RPC after error@1215043127)))
16:31:27.712 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.5.54.252:43033(127.5.54.252:43033)] disconnecting while in state READY
16:31:27.713 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.5.54.254:39175(127.5.54.254:39175)] disconnecting while in state TERMINATED
16:31:27.713 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.5.54.253:36663(127.5.54.253:36663)] disconnecting while in state TERMINATED
16:31:27.718 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
16:31:27.727 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
16:31:27.728 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.5.54.252:43033(127.5.54.252:43033)] handling channelInactive
16:31:27.729 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.54.252:43033(127.5.54.252:43033)] cleaning up while in state READY due to: connection closed
16:31:27.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:31:27.733963 5339 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task4YrI62/build/tsan/bin/kudu with pid 5699
16:31:28.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-26T16:31:28Z chronyd exiting
16:31:29.972 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
16:31:29.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:31:29.973986 4615 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task4YrI62/build/tsan/bin/kudu with pid 4896
16:31:30.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:31:30.002446 4615 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task4YrI62/build/tsan/bin/kudu with pid 5058
16:31:30.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:31:30.029253 4615 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task4YrI62/build/tsan/bin/kudu with pid 5195
16:31:30.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:31:30.063495 4615 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task4YrI62/build/tsan/bin/kudu with pid 4627
16:31:30.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:31:30.093943 4615 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task4YrI62/build/tsan/bin/kudu with pid 4697
16:31:30.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 16:31:30.125222 4615 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task4YrI62/build/tsan/bin/kudu with pid 4772
16:31:31.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-26T16:31:31Z chronyd exiting
16:31:32.411 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=23, TimeoutTracker(timeout=30000, elapsed=29075), Traces: [0ms] refreshing cache from master, [52ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.254:39175, [163ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.252:43033, [168ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.253:36663, [234ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.254:39175: Network error: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [242ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.253:36663: Network error: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [724ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.252:43033: OK, [729ms] delaying RPC due to: Service unavailable: Master config (127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [748ms] refreshing cache from master, [749ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.254:39175, [751ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.252:43033, [753ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.253:36663, [757ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.253:36663: Network error: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [760ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.254:39175: Network error: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [762ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.252:43033: OK, [763ms] delaying RPC due to: Service unavailable: Master config (127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [767ms] refreshing cache from master, [768ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.254:39175, [769ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.252:43033, [770ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.253:36663, [773ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.253:36663: Network error: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [775ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.254:39175: Network error: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [778ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.252:43033: OK, [779ms] delaying RPC due to: Service unavailable: Master config (127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [787ms] refreshing cache from master, [788ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.254:39175, [789ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.252:43033, [790ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.253:36663, [795ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.252:43033: OK, [795ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.253:36663: Network error: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [796ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.254:39175: Network error: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [798ms] delaying RPC due to: Service unavailable: Master config (127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [807ms] refreshing cache from master, [808ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.254:39175, [809ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.252:43033, [810ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.253:36663, [812ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.253:36663: Network error: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [814ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.254:39175: Network error: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [816ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.252:43033: OK, [817ms] delaying RPC due to: Service unavailable: Master config (127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [847ms] refreshing cache from master, [847ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.254:39175, [849ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.252:43033, [849ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.253:36663, [852ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.253:36663: Network error: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [855ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.252:43033: OK, [857ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.254:39175: Network error: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [858ms] delaying RPC due to: Service unavailable: Master config (127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [887ms] refreshing cache from master, [888ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.254:39175, [889ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.252:43033, [890ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.253:36663, [892ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.253:36663: Network error: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [897ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.252:43033: OK, [898ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.254:39175: Network error: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [899ms] delaying RPC due to: Service unavailable: Master config (127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [947ms] refreshing cache from master, [948ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.254:39175, [949ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.252:43033, [950ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.253:36663, [953ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.253:36663: Network error: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [954ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.254:39175: Network error: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [956ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.252:43033: OK, [957ms] delaying RPC due to: Service unavailable: Master config (127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [1047ms] refreshing cache from master, [1048ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.254:39175, [1050ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.252:43033, [1050ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.253:36663, [1053ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.253:36663: Network error: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [1055ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.254:39175: Network error: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [1057ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.252:43033: OK, [1058ms] delaying RPC due to: Service unavailable: Master config (127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [1227ms] refreshing cache from master, [1227ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.254:39175, [1228ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.252:43033, [1229ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.253:36663, [1236ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.253:36663: Network error: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [1237ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.252:43033: OK, [1238ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.254:39175: Network error: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [1239ms] delaying RPC due to: Service unavailable: Master config (127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [1988ms] refreshing cache from master, [1988ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.254:39175, [1993ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.252:43033, [1993ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.254:39175: Network error: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [1997ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.253:36663, [1999ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.252:43033: OK, [2003ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.253:36663: Network error: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [2004ms] delaying RPC due to: Service unavailable: Master config (127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [2767ms] refreshing cache from master, [2767ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.254:39175, [2768ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.252:43033, [2769ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.253:36663, [2773ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.254:39175: Network error: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [2774ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.253:36663: Network error: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [2775ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.252:43033: OK, [2775ms] delaying RPC due to: Service unavailable: Master config (127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [5147ms] refreshing cache from master, [5147ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.254:39175, [5149ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.252:43033, [5150ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.254:39175: Network error: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [5150ms] too many traces: truncated at 100 traces, deferred=Deferred@1706261723(state=PENDING, result=null, callback=(continuation of Deferred@680635496 after retry RPC after error@593135639) -> (continuation of Deferred@2085874 after retry RPC after error@2117345934) -> (continuation of Deferred@1541616588 after retry RPC after error@739119767) -> (continuation of Deferred@1383122087 after retry RPC after error@227523637) -> (continuation of Deferred@1336967963 after retry RPC after error@334932688) -> (continuation of Deferred@1198981621 after retry RPC after error@938042170) -> (continuation of Deferred@2081068679 after retry RPC after error@1510740988) -> (continuation of Deferred@621533665 after retry RPC after error@1502056137) -> (continuation of Deferred@1896973776 after retry RPC after error@1262025725) -> (continuation of Deferred@805824426 after retry RPC after error@1427359397) -> (continuation of Deferred@676460242 after retry RPC after error@607782710) -> (continuation of Deferred@730832556 after retry RPC after error@1617056662) -> (continuation of Deferred@1193463342 after retry RPC after error@861237835) -> (continuation of Deferred@1685647475 after retry RPC after error@376758665) -> (continuation of Deferred@1198923479 after retry RPC after error@1596582744) -> (continuation of Deferred@2087809958 after retry RPC after error@1295564040) -> (continuation of Deferred@986677982 after retry RPC after error@715967988) -> (continuation of Deferred@1563916100 after retry RPC after error@1441743127) -> (continuation of Deferred@334557190 after retry RPC after error@1610570977) -> (continuation of Deferred@957652237 after retry RPC after error@375166145) -> (continuation of Deferred@45622988 after retry RPC after error@663058990) -> (continuation of Deferred@979658534 after retry RPC after error@1215043127), errback=(continuation of Deferred@680635496 after retry RPC after error@593135639) -> (continuation of Deferred@2085874 after retry RPC after error@2117345934) -> (continuation of Deferred@1541616588 after retry RPC after error@739119767) -> (continuation of Deferred@1383122087 after retry RPC after error@227523637) -> (continuation of Deferred@1336967963 after retry RPC after error@334932688) -> (continuation of Deferred@1198981621 after retry RPC after error@938042170) -> (continuation of Deferred@2081068679 after retry RPC after error@1510740988) -> (continuation of Deferred@621533665 after retry RPC after error@1502056137) -> (continuation of Deferred@1896973776 after retry RPC after error@1262025725) -> (continuation of Deferred@805824426 after retry RPC after error@1427359397) -> (continuation of Deferred@676460242 after retry RPC after error@607782710) -> (continuation of Deferred@730832556 after retry RPC after error@1617056662) -> (continuation of Deferred@1193463342 after retry RPC after error@861237835) -> (continuation of Deferred@1685647475 after retry RPC after error@376758665) -> (continuation of Deferred@1198923479 after retry RPC after error@1596582744) -> (continuation of Deferred@2087809958 after retry RPC after error@1295564040) -> (continuation of Deferred@986677982 after retry RPC after error@715967988) -> (continuation of Deferred@1563916100 after retry RPC after error@1441743127) -> (continuation of Deferred@334557190 after retry RPC after error@1610570977) -> (continuation of Deferred@957652237 after retry RPC after error@375166145) -> (continuation of Deferred@45622988 after retry RPC after error@663058990) -> (continuation of Deferred@979658534 after retry RPC after error@1215043127)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.5.54.253:36663
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=23, TimeoutTracker(timeout=30000, elapsed=29075), Traces: [0ms] refreshing cache from master, [52ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.254:39175, [163ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.252:43033, [168ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.253:36663, [234ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.254:39175: Network error: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [242ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.253:36663: Network error: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [724ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.252:43033: OK, [729ms] delaying RPC due to: Service unavailable: Master config (127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [748ms] refreshing cache from master, [749ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.254:39175, [751ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.252:43033, [753ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.253:36663, [757ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.253:36663: Network error: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [760ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.254:39175: Network error: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [762ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.252:43033: OK, [763ms] delaying RPC due to: Service unavailable: Master config (127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [767ms] refreshing cache from master, [768ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.254:39175, [769ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.252:43033, [770ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.253:36663, [773ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.253:36663: Network error: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [775ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.254:39175: Network error: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [778ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.252:43033: OK, [779ms] delaying RPC due to: Service unavailable: Master config (127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [787ms] refreshing cache from master, [788ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.254:39175, [789ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.252:43033, [790ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.253:36663, [795ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.252:43033: OK, [795ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.253:36663: Network error: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [796ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.254:39175: Network error: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [798ms] delaying RPC due to: Service unavailable: Master config (127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [807ms] refreshing cache from master, [808ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.254:39175, [809ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.252:43033, [810ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.253:36663, [812ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.253:36663: Network error: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [814ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.254:39175: Network error: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [816ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.252:43033: OK, [817ms] delaying RPC due to: Service unavailable: Master config (127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [847ms] refreshing cache from master, [847ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.254:39175, [849ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.252:43033, [849ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.253:36663, [852ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.253:36663: Network error: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [855ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.252:43033: OK, [857ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.254:39175: Network error: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [858ms] delaying RPC due to: Service unavailable: Master config (127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [887ms] refreshing cache from master, [888ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.254:39175, [889ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.252:43033, [890ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.253:36663, [892ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.253:36663: Network error: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [897ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.252:43033: OK, [898ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.254:39175: Network error: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [899ms] delaying RPC due to: Service unavailable: Master config (127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [947ms] refreshing cache from master, [948ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.254:39175, [949ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.252:43033, [950ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.253:36663, [953ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.253:36663: Network error: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [954ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.254:39175: Network error: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [956ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.252:43033: OK, [957ms] delaying RPC due to: Service unavailable: Master config (127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [1047ms] refreshing cache from master, [1048ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.254:39175, [1050ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.252:43033, [1050ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.253:36663, [1053ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.253:36663: Network error: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [1055ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.254:39175: Network error: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [1057ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.252:43033: OK, [1058ms] delaying RPC due to: Service unavailable: Master config (127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [1227ms] refreshing cache from master, [1227ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.254:39175, [1228ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.252:43033, [1229ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.253:36663, [1236ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.253:36663: Network error: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [1237ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.252:43033: OK, [1238ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.254:39175: Network error: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [1239ms] delaying RPC due to: Service unavailable: Master config (127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [1988ms] refreshing cache from master, [1988ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.254:39175, [1993ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.252:43033, [1993ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.254:39175: Network error: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [1997ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.253:36663, [1999ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.252:43033: OK, [2003ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.253:36663: Network error: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [2004ms] delaying RPC due to: Service unavailable: Master config (127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [2767ms] refreshing cache from master, [2767ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.254:39175, [2768ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.252:43033, [2769ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.253:36663, [2773ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.254:39175: Network error: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [2774ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.253:36663: Network error: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [2775ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.252:43033: OK, [2775ms] delaying RPC due to: Service unavailable: Master config (127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [5147ms] refreshing cache from master, [5147ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.254:39175, [5149ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.252:43033, [5150ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.254:39175: Network error: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [5150ms] too many traces: truncated at 100 traces, deferred=Deferred@1706261723(state=PENDING, result=null, callback=(continuation of Deferred@680635496 after retry RPC after error@593135639) -> (continuation of Deferred@2085874 after retry RPC after error@2117345934) -> (continuation of Deferred@1541616588 after retry RPC after error@739119767) -> (continuation of Deferred@1383122087 after retry RPC after error@227523637) -> (continuation of Deferred@1336967963 after retry RPC after error@334932688) -> (continuation of Deferred@1198981621 after retry RPC after error@938042170) -> (continuation of Deferred@2081068679 after retry RPC after error@1510740988) -> (continuation of Deferred@621533665 after retry RPC after error@1502056137) -> (continuation of Deferred@1896973776 after retry RPC after error@1262025725) -> (continuation of Deferred@805824426 after retry RPC after error@1427359397) -> (continuation of Deferred@676460242 after retry RPC after error@607782710) -> (continuation of Deferred@730832556 after retry RPC after error@1617056662) -> (continuation of Deferred@1193463342 after retry RPC after error@861237835) -> (continuation of Deferred@1685647475 after retry RPC after error@376758665) -> (continuation of Deferred@1198923479 after retry RPC after error@1596582744) -> (continuation of Deferred@2087809958 after retry RPC after error@1295564040) -> (continuation of Deferred@986677982 after retry RPC after error@715967988) -> (continuation of Deferred@1563916100 after retry RPC after error@1441743127) -> (continuation of Deferred@334557190 after retry RPC after error@1610570977) -> (continuation of Deferred@957652237 after retry RPC after error@375166145) -> (continuation of Deferred@45622988 after retry RPC after error@663058990) -> (continuation of Deferred@979658534 after retry RPC after error@1215043127), errback=(continuation of Deferred@680635496 after retry RPC after error@593135639) -> (continuation of Deferred@2085874 after retry RPC after error@2117345934) -> (continuation of Deferred@1541616588 after retry RPC after error@739119767) -> (continuation of Deferred@1383122087 after retry RPC after error@227523637) -> (continuation of Deferred@1336967963 after retry RPC after error@334932688) -> (continuation of Deferred@1198981621 after retry RPC after error@938042170) -> (continuation of Deferred@2081068679 after retry RPC after error@1510740988) -> (continuation of Deferred@621533665 after retry RPC after error@1502056137) -> (continuation of Deferred@1896973776 after retry RPC after error@1262025725) -> (continuation of Deferred@805824426 after retry RPC after error@1427359397) -> (continuation of Deferred@676460242 after retry RPC after error@607782710) -> (continuation of Deferred@730832556 after retry RPC after error@1617056662) -> (continuation of Deferred@1193463342 after retry RPC after error@861237835) -> (continuation of Deferred@1685647475 after retry RPC after error@376758665) -> (continuation of Deferred@1198923479 after retry RPC after error@1596582744) -> (continuation of Deferred@2087809958 after retry RPC after error@1295564040) -> (continuation of Deferred@986677982 after retry RPC after error@715967988) -> (continuation of Deferred@1563916100 after retry RPC after error@1441743127) -> (continuation of Deferred@334557190 after retry RPC after error@1610570977) -> (continuation of Deferred@957652237 after retry RPC after error@375166145) -> (continuation of Deferred@45622988 after retry RPC after error@663058990) -> (continuation of Deferred@979658534 after retry RPC after error@1215043127)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.5.54.253:36663
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
16:31:32.839 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
16:31:32.839 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=23, TimeoutTracker(timeout=30000, elapsed=29075), Traces: [0ms] refreshing cache from master, [52ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.254:39175, [163ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.252:43033, [168ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.253:36663, [234ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.254:39175: Network error: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [242ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.253:36663: Network error: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [724ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.252:43033: OK, [729ms] delaying RPC due to: Service unavailable: Master config (127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [748ms] refreshing cache from master, [749ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.254:39175, [751ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.252:43033, [753ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.253:36663, [757ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.253:36663: Network error: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [760ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.254:39175: Network error: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [762ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.252:43033: OK, [763ms] delaying RPC due to: Service unavailable: Master config (127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [767ms] refreshing cache from master, [768ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.254:39175, [769ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.252:43033, [770ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.253:36663, [773ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.253:36663: Network error: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [775ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.254:39175: Network error: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [778ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.252:43033: OK, [779ms] delaying RPC due to: Service unavailable: Master config (127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [787ms] refreshing cache from master, [788ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.254:39175, [789ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.252:43033, [790ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.253:36663, [795ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.252:43033: OK, [795ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.253:36663: Network error: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [796ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.254:39175: Network error: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [798ms] delaying RPC due to: Service unavailable: Master config (127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [807ms] refreshing cache from master, [808ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.254:39175, [809ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.252:43033, [810ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.253:36663, [812ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.253:36663: Network error: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [814ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.254:39175: Network error: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [816ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.252:43033: OK, [817ms] delaying RPC due to: Service unavailable: Master config (127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [847ms] refreshing cache from master, [847ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.254:39175, [849ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.252:43033, [849ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.253:36663, [852ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.253:36663: Network error: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [855ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.252:43033: OK, [857ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.254:39175: Network error: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [858ms] delaying RPC due to: Service unavailable: Master config (127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [887ms] refreshing cache from master, [888ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.254:39175, [889ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.252:43033, [890ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.253:36663, [892ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.253:36663: Network error: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [897ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.252:43033: OK, [898ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.254:39175: Network error: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [899ms] delaying RPC due to: Service unavailable: Master config (127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [947ms] refreshing cache from master, [948ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.254:39175, [949ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.252:43033, [950ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.253:36663, [953ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.253:36663: Network error: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [954ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.254:39175: Network error: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [956ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.252:43033: OK, [957ms] delaying RPC due to: Service unavailable: Master config (127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [1047ms] refreshing cache from master, [1048ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.254:39175, [1050ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.252:43033, [1050ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.253:36663, [1053ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.253:36663: Network error: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [1055ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.254:39175: Network error: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [1057ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.252:43033: OK, [1058ms] delaying RPC due to: Service unavailable: Master config (127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [1227ms] refreshing cache from master, [1227ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.254:39175, [1228ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.252:43033, [1229ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.253:36663, [1236ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.253:36663: Network error: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [1237ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.252:43033: OK, [1238ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.254:39175: Network error: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [1239ms] delaying RPC due to: Service unavailable: Master config (127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [1988ms] refreshing cache from master, [1988ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.254:39175, [1993ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.252:43033, [1993ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.254:39175: Network error: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [1997ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.253:36663, [1999ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.252:43033: OK, [2003ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.253:36663: Network error: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [2004ms] delaying RPC due to: Service unavailable: Master config (127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [2767ms] refreshing cache from master, [2767ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.254:39175, [2768ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.252:43033, [2769ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.253:36663, [2773ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.254:39175: Network error: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [2774ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.253:36663: Network error: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [2775ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.252:43033: OK, [2775ms] delaying RPC due to: Service unavailable: Master config (127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [5147ms] refreshing cache from master, [5147ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.254:39175, [5149ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.252:43033, [5150ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.254:39175: Network error: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [5150ms] too many traces: truncated at 100 traces, deferred=Deferred@1706261723(state=PENDING, result=null, callback=(continuation of Deferred@680635496 after retry RPC after error@593135639) -> (continuation of Deferred@2085874 after retry RPC after error@2117345934) -> (continuation of Deferred@1541616588 after retry RPC after error@739119767) -> (continuation of Deferred@1383122087 after retry RPC after error@227523637) -> (continuation of Deferred@1336967963 after retry RPC after error@334932688) -> (continuation of Deferred@1198981621 after retry RPC after error@938042170) -> (continuation of Deferred@2081068679 after retry RPC after error@1510740988) -> (continuation of Deferred@621533665 after retry RPC after error@1502056137) -> (continuation of Deferred@1896973776 after retry RPC after error@1262025725) -> (continuation of Deferred@805824426 after retry RPC after error@1427359397) -> (continuation of Deferred@676460242 after retry RPC after error@607782710) -> (continuation of Deferred@730832556 after retry RPC after error@1617056662) -> (continuation of Deferred@1193463342 after retry RPC after error@861237835) -> (continuation of Deferred@1685647475 after retry RPC after error@376758665) -> (continuation of Deferred@1198923479 after retry RPC after error@1596582744) -> (continuation of Deferred@2087809958 after retry RPC after error@1295564040) -> (continuation of Deferred@986677982 after retry RPC after error@715967988) -> (continuation of Deferred@1563916100 after retry RPC after error@1441743127) -> (continuation of Deferred@334557190 after retry RPC after error@1610570977) -> (continuation of Deferred@957652237 after retry RPC after error@375166145) -> (continuation of Deferred@45622988 after retry RPC after error@663058990) -> (continuation of Deferred@979658534 after retry RPC after error@1215043127), errback=(continuation of Deferred@680635496 after retry RPC after error@593135639) -> (continuation of Deferred@2085874 after retry RPC after error@2117345934) -> (continuation of Deferred@1541616588 after retry RPC after error@739119767) -> (continuation of Deferred@1383122087 after retry RPC after error@227523637) -> (continuation of Deferred@1336967963 after retry RPC after error@334932688) -> (continuation of Deferred@1198981621 after retry RPC after error@938042170) -> (continuation of Deferred@2081068679 after retry RPC after error@1510740988) -> (continuation of Deferred@621533665 after retry RPC after error@1502056137) -> (continuation of Deferred@1896973776 after retry RPC after error@1262025725) -> (continuation of Deferred@805824426 after retry RPC after error@1427359397) -> (continuation of Deferred@676460242 after retry RPC after error@607782710) -> (continuation of Deferred@730832556 after retry RPC after error@1617056662) -> (continuation of Deferred@1193463342 after retry RPC after error@861237835) -> (continuation of Deferred@1685647475 after retry RPC after error@376758665) -> (continuation of Deferred@1198923479 after retry RPC after error@1596582744) -> (continuation of Deferred@2087809958 after retry RPC after error@1295564040) -> (continuation of Deferred@986677982 after retry RPC after error@715967988) -> (continuation of Deferred@1563916100 after retry RPC after error@1441743127) -> (continuation of Deferred@334557190 after retry RPC after error@1610570977) -> (continuation of Deferred@957652237 after retry RPC after error@375166145) -> (continuation of Deferred@45622988 after retry RPC after error@663058990) -> (continuation of Deferred@979658534 after retry RPC after error@1215043127)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.5.54.253:36663
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=23, TimeoutTracker(timeout=30000, elapsed=29075), Traces: [0ms] refreshing cache from master, [52ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.254:39175, [163ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.252:43033, [168ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.253:36663, [234ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.254:39175: Network error: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [242ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.253:36663: Network error: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [724ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.252:43033: OK, [729ms] delaying RPC due to: Service unavailable: Master config (127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [748ms] refreshing cache from master, [749ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.254:39175, [751ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.252:43033, [753ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.253:36663, [757ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.253:36663: Network error: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [760ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.254:39175: Network error: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [762ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.252:43033: OK, [763ms] delaying RPC due to: Service unavailable: Master config (127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [767ms] refreshing cache from master, [768ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.254:39175, [769ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.252:43033, [770ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.253:36663, [773ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.253:36663: Network error: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [775ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.254:39175: Network error: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [778ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.252:43033: OK, [779ms] delaying RPC due to: Service unavailable: Master config (127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [787ms] refreshing cache from master, [788ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.254:39175, [789ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.252:43033, [790ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.253:36663, [795ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.252:43033: OK, [795ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.253:36663: Network error: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [796ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.254:39175: Network error: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [798ms] delaying RPC due to: Service unavailable: Master config (127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [807ms] refreshing cache from master, [808ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.254:39175, [809ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.252:43033, [810ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.253:36663, [812ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.253:36663: Network error: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [814ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.254:39175: Network error: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [816ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.252:43033: OK, [817ms] delaying RPC due to: Service unavailable: Master config (127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [847ms] refreshing cache from master, [847ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.254:39175, [849ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.252:43033, [849ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.253:36663, [852ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.253:36663: Network error: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [855ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.252:43033: OK, [857ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.254:39175: Network error: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [858ms] delaying RPC due to: Service unavailable: Master config (127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [887ms] refreshing cache from master, [888ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.254:39175, [889ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.252:43033, [890ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.253:36663, [892ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.253:36663: Network error: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [897ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.252:43033: OK, [898ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.254:39175: Network error: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [899ms] delaying RPC due to: Service unavailable: Master config (127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [947ms] refreshing cache from master, [948ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.254:39175, [949ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.252:43033, [950ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.253:36663, [953ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.253:36663: Network error: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [954ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.254:39175: Network error: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [956ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.252:43033: OK, [957ms] delaying RPC due to: Service unavailable: Master config (127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [1047ms] refreshing cache from master, [1048ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.254:39175, [1050ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.252:43033, [1050ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.253:36663, [1053ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.253:36663: Network error: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [1055ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.254:39175: Network error: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [1057ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.252:43033: OK, [1058ms] delaying RPC due to: Service unavailable: Master config (127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [1227ms] refreshing cache from master, [1227ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.254:39175, [1228ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.252:43033, [1229ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.253:36663, [1236ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.253:36663: Network error: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [1237ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.252:43033: OK, [1238ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.254:39175: Network error: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [1239ms] delaying RPC due to: Service unavailable: Master config (127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [1988ms] refreshing cache from master, [1988ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.254:39175, [1993ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.252:43033, [1993ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.254:39175: Network error: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [1997ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.253:36663, [1999ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.252:43033: OK, [2003ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.253:36663: Network error: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [2004ms] delaying RPC due to: Service unavailable: Master config (127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [2767ms] refreshing cache from master, [2767ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.254:39175, [2768ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.252:43033, [2769ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.253:36663, [2773ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.254:39175: Network error: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [2774ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.253:36663: Network error: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [2775ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.252:43033: OK, [2775ms] delaying RPC due to: Service unavailable: Master config (127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663, [5147ms] refreshing cache from master, [5147ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.254:39175, [5149ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.54.252:43033, [5150ms] Sub RPC ConnectToMaster: received response from server master-127.5.54.254:39175: Network error: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175, [5150ms] too many traces: truncated at 100 traces, deferred=Deferred@1706261723(state=PENDING, result=null, callback=(continuation of Deferred@680635496 after retry RPC after error@593135639) -> (continuation of Deferred@2085874 after retry RPC after error@2117345934) -> (continuation of Deferred@1541616588 after retry RPC after error@739119767) -> (continuation of Deferred@1383122087 after retry RPC after error@227523637) -> (continuation of Deferred@1336967963 after retry RPC after error@334932688) -> (continuation of Deferred@1198981621 after retry RPC after error@938042170) -> (continuation of Deferred@2081068679 after retry RPC after error@1510740988) -> (continuation of Deferred@621533665 after retry RPC after error@1502056137) -> (continuation of Deferred@1896973776 after retry RPC after error@1262025725) -> (continuation of Deferred@805824426 after retry RPC after error@1427359397) -> (continuation of Deferred@676460242 after retry RPC after error@607782710) -> (continuation of Deferred@730832556 after retry RPC after error@1617056662) -> (continuation of Deferred@1193463342 after retry RPC after error@861237835) -> (continuation of Deferred@1685647475 after retry RPC after error@376758665) -> (continuation of Deferred@1198923479 after retry RPC after error@1596582744) -> (continuation of Deferred@2087809958 after retry RPC after error@1295564040) -> (continuation of Deferred@986677982 after retry RPC after error@715967988) -> (continuation of Deferred@1563916100 after retry RPC after error@1441743127) -> (continuation of Deferred@334557190 after retry RPC after error@1610570977) -> (continuation of Deferred@957652237 after retry RPC after error@375166145) -> (continuation of Deferred@45622988 after retry RPC after error@663058990) -> (continuation of Deferred@979658534 after retry RPC after error@1215043127), errback=(continuation of Deferred@680635496 after retry RPC after error@593135639) -> (continuation of Deferred@2085874 after retry RPC after error@2117345934) -> (continuation of Deferred@1541616588 after retry RPC after error@739119767) -> (continuation of Deferred@1383122087 after retry RPC after error@227523637) -> (continuation of Deferred@1336967963 after retry RPC after error@334932688) -> (continuation of Deferred@1198981621 after retry RPC after error@938042170) -> (continuation of Deferred@2081068679 after retry RPC after error@1510740988) -> (continuation of Deferred@621533665 after retry RPC after error@1502056137) -> (continuation of Deferred@1896973776 after retry RPC after error@1262025725) -> (continuation of Deferred@805824426 after retry RPC after error@1427359397) -> (continuation of Deferred@676460242 after retry RPC after error@607782710) -> (continuation of Deferred@730832556 after retry RPC after error@1617056662) -> (continuation of Deferred@1193463342 after retry RPC after error@861237835) -> (continuation of Deferred@1685647475 after retry RPC after error@376758665) -> (continuation of Deferred@1198923479 after retry RPC after error@1596582744) -> (continuation of Deferred@2087809958 after retry RPC after error@1295564040) -> (continuation of Deferred@986677982 after retry RPC after error@715967988) -> (continuation of Deferred@1563916100 after retry RPC after error@1441743127) -> (continuation of Deferred@334557190 after retry RPC after error@1610570977) -> (continuation of Deferred@957652237 after retry RPC after error@375166145) -> (continuation of Deferred@45622988 after retry RPC after error@663058990) -> (continuation of Deferred@979658534 after retry RPC after error@1215043127)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.5.54.254:39175,127.5.54.252:43033,127.5.54.253:36663) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.254:39175(127.5.54.254:39175): Connection refused: /127.5.54.254:39175,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.54.253:36663(127.5.54.253:36663): Connection refused: /127.5.54.253:36663
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.5.54.253:36663
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)