10:27:45.817 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
10:27:45.834 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-tasks36zsq/test-tmp
10:27:45.838 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
10:27:45.843 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-tasks36zsq/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
10:27:46.098 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
10:27:46.201 [DEBUG - main] (MiniKuduCluster.java:195) Response:
10:27:46.203 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
10:27:46.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:46.206434 8886 mini_oidc.cc:152] Starting JWKS server
10:27:46.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:46.210471 8886 webserver.cc:460] Webserver started at http://127.0.0.1:46759/ using document root <none> and password file <none>
10:27:46.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:46.211114 8886 mini_oidc.cc:189] Starting OIDC Discovery server
10:27:46.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:46.213006 8886 webserver.cc:460] Webserver started at http://127.0.0.1:33809/ using document root <none> and password file <none>
10:27:46.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T10:27:46Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
10:27:46.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T10:27:46Z Disabled control of system clock
10:27:46.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:46.263938 8886 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasks36zsq/build/tsan/bin/kudu
10:27:46.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasks36zsq/build/tsan/bin/kudu
10:27:46.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/master-0/wal
10:27:46.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/master-0/data
10:27:46.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:27:46.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:27:46.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:27:46.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:27:46.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:27:46.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:27:46.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/master-0/logs
10:27:46.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/master-0/data/info.pb
10:27:46.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:27:46.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:27:46.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:27:46.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:27:46.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:27:46.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:27:46.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:27:46.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:27:46.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
10:27:46.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:27:46.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:27:46.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:27:46.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.173.190:41013
10:27:46.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.173.190
10:27:46.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:27:46.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.173.148:33799
10:27:46.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:27:46.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:27:46.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:27:46.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:27:46.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.173.190:41013,127.8.173.189:41829,127.8.173.188:43959
10:27:46.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:27:46.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46759/jwks.json with env {}
10:27:46.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:46.549881 8898 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
10:27:46.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:46.550396 8898 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:27:46.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:46.551074 8898 flags.cc:425] Enabled unsafe flag: --never_fsync=true
10:27:46.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:46.580832 8898 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
10:27:46.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:46.581086 8898 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
10:27:46.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:46.581270 8898 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
10:27:46.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:46.581557 8898 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
10:27:46.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:46.581862 8898 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
10:27:46.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:46.582095 8898 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46759/jwks.json
10:27:46.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:46.615288 8898 master_runner.cc:386] Master server non-default flags:
10:27:46.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:27:46.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.173.148:33799
10:27:46.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:27:46.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:27:46.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/master-0/data
10:27:46.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/master-0/wal
10:27:46.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:27:46.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.173.190:41013,127.8.173.189:41829,127.8.173.188:43959
10:27:46.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:27:46.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:27:46.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:27:46.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.173.190:41013
10:27:46.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:27:46.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:27:46.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:27:46.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46759/jwks.json
10:27:46.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:27:46.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:27:46.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/master-0/data/info.pb
10:27:46.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.173.190
10:27:46.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:27:46.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:27:46.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:27:46.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:27:46.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:27:46.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:27:46.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/master-0/logs
10:27:46.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:27:46.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:27:46.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
10:27:46.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
10:27:46.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
10:27:46.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
10:27:46.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:27:46.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 09:27:27 UTC on ccycloud.quasar-stwief.root.comops.site
10:27:46.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679491
10:27:46.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
10:27:46.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:46.616947 8898 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:27:46.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:46.618490 8898 file_cache.cc:493] Constructed file cache file cache with capacity 419430
10:27:46.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:46.637410 8904 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:47.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:46.639269 8905 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:47.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:46.637454 8907 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:47.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:47.816907 8906 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
10:27:47.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:47.816982 8898 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
10:27:47.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:47.821401 8898 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:27:47.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:47.824528 8898 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:27:47.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:47.825976 8898 hybrid_clock.cc:648] HybridClock initialized: now 1790677667825890 us; error 81 us; skew 500 ppm
10:27:47.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:47.827039 8898 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:27:47.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:47.837654 8898 webserver.cc:460] Webserver started at http://127.8.173.190:41023/ using document root <none> and password file <none>
10:27:47.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:47.838660 8898 fs_manager.cc:359] Metadata directory not provided
10:27:47.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:47.838897 8898 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:27:47.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:47.839453 8898 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
10:27:47.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:47.844431 8898 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/master-0/data/instance:
10:27:47.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9753a1cbb2d54292aeac0d805e240cf0"
10:27:47.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 10:27:47 on dist-test-slave-9qlq"
10:27:47.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:47.845696 8898 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/master-0/wal/instance:
10:27:47.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9753a1cbb2d54292aeac0d805e240cf0"
10:27:47.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 10:27:47 on dist-test-slave-9qlq"
10:27:47.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:47.852699 8898 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.002s sys 0.005s
10:27:47.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:47.857826 8914 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
10:27:47.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:47.858948 8898 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.003s
10:27:47.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:47.859202 8898 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/master-0/data,/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/master-0/wal
10:27:47.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9753a1cbb2d54292aeac0d805e240cf0"
10:27:47.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 10:27:47 on dist-test-slave-9qlq"
10:27:47.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:47.859498 8898 fs_report.cc:383] FS layout report
10:27:47.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:27:47.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/master-0/wal
10:27:47.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/master-0/wal
10:27:47.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/master-0/data/data
10:27:47.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:27:47.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:27:47.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:27:47.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:27:47.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:27:47.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:27:47.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:27:47.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:27:47.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:27:47.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:27:47.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:27:47.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:47.891927 8898 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
10:27:47.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:47.893273 8898 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:27:47.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:47.893664 8898 kserver.cc:163] Server-wide thread pool size limit: 3276
10:27:47.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:47.962085 8898 rpc_server.cc:307] RPC server started. Bound to: 127.8.173.190:41013
10:27:47.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:47.962185 8965 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.8.173.190:41013 every 8 connection(s)
10:27:47.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:47.964691 8898 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/master-0/data/info.pb
10:27:47.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:47.969599 8966 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
10:27:47.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:47.971182 8886 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasks36zsq/build/tsan/bin/kudu as pid 8898
10:27:47.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:47.971735 8886 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/master-0/wal/instance
10:27:47.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:47.976774 8886 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasks36zsq/build/tsan/bin/kudu
10:27:47.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasks36zsq/build/tsan/bin/kudu
10:27:47.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/master-1/wal
10:27:47.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/master-1/data
10:27:47.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:27:47.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:27:47.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:27:47.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:27:47.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:27:47.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:27:47.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/master-1/logs
10:27:47.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/master-1/data/info.pb
10:27:47.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:27:47.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:27:47.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:27:47.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:27:47.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:27:47.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:27:47.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:27:47.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:27:47.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
10:27:47.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:27:47.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:27:47.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:27:47.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.173.189:41829
10:27:47.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.173.189
10:27:47.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:27:47.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.173.148:33799
10:27:47.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:27:47.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:27:47.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:27:47.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:27:47.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.173.190:41013,127.8.173.189:41829,127.8.173.188:43959
10:27:47.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:27:47.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46759/jwks.json with env {}
10:27:47.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:47.985265 8966 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.173.190" port: 41013 } has no permanent_uuid. Determining permanent_uuid...
10:27:48.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:48.012465 8966 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.173.189" port: 41829 } has no permanent_uuid. Determining permanent_uuid...
10:27:48.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:48.014725 8917 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.173.189:41829: connect: Connection refused (error 111)
10:27:48.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:48.018757 8966 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.173.189:41829: Network error: Client connection negotiation failed: client connection to 127.8.173.189:41829: connect: Connection refused (error 111)
10:27:48.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:48.068286 8966 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.173.189" port: 41829 } attempt: 1
10:27:48.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:48.072114 8966 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.173.189:41829: Network error: Client connection negotiation failed: client connection to 127.8.173.189:41829: connect: Connection refused (error 111)
10:27:48.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:48.140614 8966 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.173.189" port: 41829 } attempt: 2
10:27:48.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:48.144683 8966 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.173.189:41829: Network error: Client connection negotiation failed: client connection to 127.8.173.189:41829: connect: Connection refused (error 111)
10:27:48.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:48.236068 8966 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.173.189" port: 41829 } attempt: 3
10:27:48.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:48.240056 8966 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.173.189:41829: Network error: Client connection negotiation failed: client connection to 127.8.173.189:41829: connect: Connection refused (error 111)
10:27:48.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:48.272188 8968 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
10:27:48.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:48.272774 8968 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:27:48.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:48.273331 8968 flags.cc:425] Enabled unsafe flag: --never_fsync=true
10:27:48.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:48.302707 8968 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
10:27:48.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:48.303056 8968 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
10:27:48.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:48.303377 8968 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
10:27:48.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:48.303737 8968 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
10:27:48.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:48.304076 8968 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
10:27:48.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:48.304293 8968 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46759/jwks.json
10:27:48.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:48.338912 8968 master_runner.cc:386] Master server non-default flags:
10:27:48.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:27:48.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.173.148:33799
10:27:48.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:27:48.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:27:48.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/master-1/data
10:27:48.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/master-1/wal
10:27:48.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:27:48.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.173.190:41013,127.8.173.189:41829,127.8.173.188:43959
10:27:48.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:27:48.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:27:48.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:27:48.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.173.189:41829
10:27:48.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:27:48.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:27:48.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:27:48.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46759/jwks.json
10:27:48.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:27:48.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:27:48.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/master-1/data/info.pb
10:27:48.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.173.189
10:27:48.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:27:48.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:27:48.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:27:48.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:27:48.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:27:48.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:27:48.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/master-1/logs
10:27:48.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:27:48.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:27:48.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
10:27:48.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
10:27:48.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
10:27:48.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
10:27:48.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:27:48.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 09:27:27 UTC on ccycloud.quasar-stwief.root.comops.site
10:27:48.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679491
10:27:48.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
10:27:48.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:48.340277 8968 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:27:48.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:48.342417 8968 file_cache.cc:493] Constructed file cache file cache with capacity 419430
10:27:48.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:48.359378 8977 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:48.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:48.383455 8966 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.173.189" port: 41829 } attempt: 4
10:27:48.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:48.388151 8966 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.173.189:41829: Network error: Client connection negotiation failed: client connection to 127.8.173.189:41829: connect: Connection refused (error 111)
10:27:48.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:48.687603 8966 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.173.189" port: 41829 } attempt: 5
10:27:48.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:48.691720 8966 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.173.189:41829: Network error: Client connection negotiation failed: client connection to 127.8.173.189:41829: connect: Connection refused (error 111)
10:27:49.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:49.239108 8966 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.173.189" port: 41829 } attempt: 6
10:27:49.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:49.246802 8966 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.173.189:41829: Network error: Client connection negotiation failed: client connection to 127.8.173.189:41829: connect: Connection refused (error 111)
10:27:49.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:48.359871 8980 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:49.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:48.360034 8978 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:49.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:48.361168 8968 server_base.cc:1034] running on GCE node
10:27:49.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:49.549678 8968 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:27:49.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:49.552678 8968 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:27:49.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:49.554041 8968 hybrid_clock.cc:648] HybridClock initialized: now 1790677669554025 us; error 54 us; skew 500 ppm
10:27:49.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:49.554849 8968 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:27:49.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:49.561465 8968 webserver.cc:460] Webserver started at http://127.8.173.189:38431/ using document root <none> and password file <none>
10:27:49.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:49.562345 8968 fs_manager.cc:359] Metadata directory not provided
10:27:49.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:49.562525 8968 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:27:49.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:49.562916 8968 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
10:27:49.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:49.567729 8968 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/master-1/data/instance:
10:27:49.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5a5ddfff0bfc4646a4c8c88324092bbd"
10:27:49.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 10:27:49 on dist-test-slave-9qlq"
10:27:49.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:49.568928 8968 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/master-1/wal/instance:
10:27:49.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5a5ddfff0bfc4646a4c8c88324092bbd"
10:27:49.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 10:27:49 on dist-test-slave-9qlq"
10:27:49.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:49.576007 8968 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.006s sys 0.000s
10:27:49.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:49.581570 8989 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
10:27:49.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:49.582623 8968 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.000s sys 0.005s
10:27:49.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:49.582883 8968 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/master-1/data,/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/master-1/wal
10:27:49.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5a5ddfff0bfc4646a4c8c88324092bbd"
10:27:49.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 10:27:49 on dist-test-slave-9qlq"
10:27:49.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:49.583164 8968 fs_report.cc:383] FS layout report
10:27:49.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:27:49.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/master-1/wal
10:27:49.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/master-1/wal
10:27:49.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/master-1/data/data
10:27:49.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:27:49.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:27:49.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:27:49.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:27:49.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:27:49.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:27:49.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:27:49.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:27:49.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:27:49.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:27:49.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:27:49.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:49.616935 8968 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
10:27:49.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:49.618341 8968 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:27:49.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:49.618716 8968 kserver.cc:163] Server-wide thread pool size limit: 3276
10:27:49.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:49.684929 8968 rpc_server.cc:307] RPC server started. Bound to: 127.8.173.189:41829
10:27:49.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:49.685010 9040 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.8.173.189:41829 every 8 connection(s)
10:27:49.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:49.687681 8968 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/master-1/data/info.pb
10:27:49.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:49.692543 9041 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
10:27:49.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:49.696089 8886 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasks36zsq/build/tsan/bin/kudu as pid 8968
10:27:49.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:49.696545 8886 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/master-1/wal/instance
10:27:49.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:49.698607 8886 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasks36zsq/build/tsan/bin/kudu
10:27:49.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasks36zsq/build/tsan/bin/kudu
10:27:49.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/master-2/wal
10:27:49.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/master-2/data
10:27:49.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:27:49.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:27:49.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:27:49.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:27:49.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:27:49.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:27:49.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/master-2/logs
10:27:49.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/master-2/data/info.pb
10:27:49.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:27:49.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:27:49.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:27:49.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:27:49.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:27:49.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:27:49.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:27:49.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:27:49.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
10:27:49.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:27:49.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:27:49.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:27:49.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.173.188:43959
10:27:49.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.173.188
10:27:49.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:27:49.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.173.148:33799
10:27:49.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:27:49.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:27:49.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:27:49.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:27:49.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.173.190:41013,127.8.173.189:41829,127.8.173.188:43959
10:27:49.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:27:49.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46759/jwks.json with env {}
10:27:49.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:49.703806 9041 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.173.190" port: 41013 } has no permanent_uuid. Determining permanent_uuid...
10:27:49.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:49.727980 9041 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.173.189" port: 41829 } has no permanent_uuid. Determining permanent_uuid...
10:27:49.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:49.744879 9041 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.173.188" port: 43959 } has no permanent_uuid. Determining permanent_uuid...
10:27:49.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:49.747635 8990 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.173.188:43959: connect: Connection refused (error 111)
10:27:49.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:49.751624 9041 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.173.188:43959: Network error: Client connection negotiation failed: client connection to 127.8.173.188:43959: connect: Connection refused (error 111)
10:27:49.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:49.801185 9041 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.173.188" port: 43959 } attempt: 1
10:27:49.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:49.805284 9041 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.173.188:43959: Network error: Client connection negotiation failed: client connection to 127.8.173.188:43959: connect: Connection refused (error 111)
10:27:49.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:49.873687 9041 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.173.188" port: 43959 } attempt: 2
10:27:49.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:49.877187 9041 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.173.188:43959: Network error: Client connection negotiation failed: client connection to 127.8.173.188:43959: connect: Connection refused (error 111)
10:27:49.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:49.968629 9041 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.173.188" port: 43959 } attempt: 3
10:27:49.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:49.972833 9041 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.173.188:43959: Network error: Client connection negotiation failed: client connection to 127.8.173.188:43959: connect: Connection refused (error 111)
10:27:50.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:50.007714 9043 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
10:27:50.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:50.008424 9043 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:27:50.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:50.009084 9043 flags.cc:425] Enabled unsafe flag: --never_fsync=true
10:27:50.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:50.039441 9043 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
10:27:50.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:50.039790 9043 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
10:27:50.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:50.040122 9043 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
10:27:50.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:50.040477 9043 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
10:27:50.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:50.040812 9043 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
10:27:50.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:50.041056 9043 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46759/jwks.json
10:27:50.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:50.074853 9043 master_runner.cc:386] Master server non-default flags:
10:27:50.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:27:50.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.173.148:33799
10:27:50.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:27:50.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:27:50.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/master-2/data
10:27:50.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/master-2/wal
10:27:50.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:27:50.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.173.190:41013,127.8.173.189:41829,127.8.173.188:43959
10:27:50.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:27:50.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:27:50.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:27:50.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.173.188:43959
10:27:50.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:27:50.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:27:50.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:27:50.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46759/jwks.json
10:27:50.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:27:50.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:27:50.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/master-2/data/info.pb
10:27:50.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.173.188
10:27:50.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:27:50.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:27:50.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:27:50.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:27:50.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:27:50.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:27:50.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/master-2/logs
10:27:50.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:27:50.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:27:50.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
10:27:50.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
10:27:50.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
10:27:50.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
10:27:50.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:27:50.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 09:27:27 UTC on ccycloud.quasar-stwief.root.comops.site
10:27:50.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679491
10:27:50.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
10:27:50.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:50.076373 9043 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:27:50.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:50.078418 9043 file_cache.cc:493] Constructed file cache file cache with capacity 419430
10:27:50.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:50.096738 9053 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:50.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:50.116264 9041 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.173.188" port: 43959 } attempt: 4
10:27:50.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:50.121104 9041 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.173.188:43959: Network error: Client connection negotiation failed: client connection to 127.8.173.188:43959: connect: Connection refused (error 111)
10:27:50.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:50.307188 8966 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.173.189" port: 41829 } attempt: 7
10:27:50.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:50.319550 8966 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.173.188" port: 43959 } has no permanent_uuid. Determining permanent_uuid...
10:27:50.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:50.324671 8966 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.173.188:43959: Network error: Client connection negotiation failed: client connection to 127.8.173.188:43959: connect: Connection refused (error 111)
10:27:50.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:50.383292 8966 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.173.188" port: 43959 } attempt: 1
10:27:50.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:50.387208 8966 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.173.188:43959: Network error: Client connection negotiation failed: client connection to 127.8.173.188:43959: connect: Connection refused (error 111)
10:27:50.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:50.420481 9041 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.173.188" port: 43959 } attempt: 5
10:27:50.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:50.425464 9041 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.173.188:43959: Network error: Client connection negotiation failed: client connection to 127.8.173.188:43959: connect: Connection refused (error 111)
10:27:50.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:50.468803 8966 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.173.188" port: 43959 } attempt: 2
10:27:50.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:50.472854 8966 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.173.188:43959: Network error: Client connection negotiation failed: client connection to 127.8.173.188:43959: connect: Connection refused (error 111)
10:27:50.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:50.558493 8966 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.173.188" port: 43959 } attempt: 3
10:27:50.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:50.562503 8966 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.173.188:43959: Network error: Client connection negotiation failed: client connection to 127.8.173.188:43959: connect: Connection refused (error 111)
10:27:50.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:50.702931 8966 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.173.188" port: 43959 } attempt: 4
10:27:50.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:50.707443 8966 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.173.188:43959: Network error: Client connection negotiation failed: client connection to 127.8.173.188:43959: connect: Connection refused (error 111)
10:27:50.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:50.972822 9041 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.173.188" port: 43959 } attempt: 6
10:27:50.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:50.979358 9041 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.173.188:43959: Network error: Client connection negotiation failed: client connection to 127.8.173.188:43959: connect: Connection refused (error 111)
10:27:50.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:50.990809 8966 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.173.188" port: 43959 } attempt: 5
10:27:50.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:50.995097 8966 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.173.188:43959: Network error: Client connection negotiation failed: client connection to 127.8.173.188:43959: connect: Connection refused (error 111)
10:27:51.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:50.096876 9056 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:51.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:50.097084 9054 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:51.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:50.098201 9043 server_base.cc:1034] running on GCE node
10:27:51.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.297408 9043 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:27:51.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.299962 9043 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:27:51.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.301342 9043 hybrid_clock.cc:648] HybridClock initialized: now 1790677671301313 us; error 38 us; skew 500 ppm
10:27:51.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.302268 9043 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:27:51.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.308492 9043 webserver.cc:460] Webserver started at http://127.8.173.188:37265/ using document root <none> and password file <none>
10:27:51.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.309463 9043 fs_manager.cc:359] Metadata directory not provided
10:27:51.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.309701 9043 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:27:51.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.310232 9043 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
10:27:51.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.315055 9043 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/master-2/data/instance:
10:27:51.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "570b75c366184198a09749ea161d487c"
10:27:51.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 10:27:51 on dist-test-slave-9qlq"
10:27:51.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.316188 9043 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/master-2/wal/instance:
10:27:51.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "570b75c366184198a09749ea161d487c"
10:27:51.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 10:27:51 on dist-test-slave-9qlq"
10:27:51.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.323688 9043 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
10:27:51.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.329108 9068 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
10:27:51.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.330080 9043 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.002s
10:27:51.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.330413 9043 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/master-2/data,/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/master-2/wal
10:27:51.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "570b75c366184198a09749ea161d487c"
10:27:51.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 10:27:51 on dist-test-slave-9qlq"
10:27:51.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.330770 9043 fs_report.cc:383] FS layout report
10:27:51.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:27:51.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/master-2/wal
10:27:51.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/master-2/wal
10:27:51.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/master-2/data/data
10:27:51.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:27:51.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:27:51.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:27:51.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:27:51.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:27:51.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:27:51.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:27:51.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:27:51.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:27:51.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:27:51.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:27:51.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.370991 9043 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
10:27:51.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.372596 9043 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:27:51.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.373095 9043 kserver.cc:163] Server-wide thread pool size limit: 3276
10:27:51.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.441965 9043 rpc_server.cc:307] RPC server started. Bound to: 127.8.173.188:43959
10:27:51.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.442068 9119 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.8.173.188:43959 every 8 connection(s)
10:27:51.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.444728 9043 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/master-2/data/info.pb
10:27:51.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.449229 8886 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasks36zsq/build/tsan/bin/kudu as pid 9043
10:27:51.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.449687 8886 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/master-2/wal/instance
10:27:51.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.451655 9120 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
10:27:51.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.468276 9120 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.173.190" port: 41013 } has no permanent_uuid. Determining permanent_uuid...
10:27:51.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.493129 9120 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.173.189" port: 41829 } has no permanent_uuid. Determining permanent_uuid...
10:27:51.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.504623 9120 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.173.188" port: 43959 } has no permanent_uuid. Determining permanent_uuid...
10:27:51.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.520267 9120 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 570b75c366184198a09749ea161d487c: Bootstrap starting.
10:27:51.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.526682 9120 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 570b75c366184198a09749ea161d487c: Neither blocks nor log segments found. Creating new log.
10:27:51.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.528559 9120 log.cc:824] T 00000000000000000000000000000000 P 570b75c366184198a09749ea161d487c: Log is configured to *not* fsync() on all Append() calls
10:27:51.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.533022 9120 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 570b75c366184198a09749ea161d487c: No bootstrap required, opened a new log
10:27:51.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.547477 8966 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.173.188" port: 43959 } attempt: 6
10:27:51.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.555751 9120 raft_consensus.cc:348] T 00000000000000000000000000000000 P 570b75c366184198a09749ea161d487c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9753a1cbb2d54292aeac0d805e240cf0" member_type: VOTER last_known_addr { host: "127.8.173.190" port: 41013 } } peers { permanent_uuid: "5a5ddfff0bfc4646a4c8c88324092bbd" member_type: VOTER last_known_addr { host: "127.8.173.189" port: 41829 } } peers { permanent_uuid: "570b75c366184198a09749ea161d487c" member_type: VOTER last_known_addr { host: "127.8.173.188" port: 43959 } }
10:27:51.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.556593 9120 raft_consensus.cc:374] T 00000000000000000000000000000000 P 570b75c366184198a09749ea161d487c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
10:27:51.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.556795 9120 raft_consensus.cc:729] T 00000000000000000000000000000000 P 570b75c366184198a09749ea161d487c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 570b75c366184198a09749ea161d487c, State: Initialized, Role: FOLLOWER
10:27:51.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.557497 9120 consensus_queue.cc:260] T 00000000000000000000000000000000 P 570b75c366184198a09749ea161d487c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9753a1cbb2d54292aeac0d805e240cf0" member_type: VOTER last_known_addr { host: "127.8.173.190" port: 41013 } } peers { permanent_uuid: "5a5ddfff0bfc4646a4c8c88324092bbd" member_type: VOTER last_known_addr { host: "127.8.173.189" port: 41829 } } peers { permanent_uuid: "570b75c366184198a09749ea161d487c" member_type: VOTER last_known_addr { host: "127.8.173.188" port: 43959 } }
10:27:51.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.560493 9130 sys_catalog.cc:455] T 00000000000000000000000000000000 P 570b75c366184198a09749ea161d487c [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9753a1cbb2d54292aeac0d805e240cf0" member_type: VOTER last_known_addr { host: "127.8.173.190" port: 41013 } } peers { permanent_uuid: "5a5ddfff0bfc4646a4c8c88324092bbd" member_type: VOTER last_known_addr { host: "127.8.173.189" port: 41829 } } peers { permanent_uuid: "570b75c366184198a09749ea161d487c" member_type: VOTER last_known_addr { host: "127.8.173.188" port: 43959 } } }
10:27:51.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.561528 9130 sys_catalog.cc:458] T 00000000000000000000000000000000 P 570b75c366184198a09749ea161d487c [sys.catalog]: This master's current role is: FOLLOWER
10:27:51.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.562296 9120 sys_catalog.cc:564] T 00000000000000000000000000000000 P 570b75c366184198a09749ea161d487c [sys.catalog]: configured and running, proceeding with master startup.
10:27:51.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.569504 8966 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9753a1cbb2d54292aeac0d805e240cf0: Bootstrap starting.
10:27:51.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.574441 8966 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 9753a1cbb2d54292aeac0d805e240cf0: Neither blocks nor log segments found. Creating new log.
10:27:51.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.576148 8966 log.cc:824] T 00000000000000000000000000000000 P 9753a1cbb2d54292aeac0d805e240cf0: Log is configured to *not* fsync() on all Append() calls
10:27:51.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.581395 8966 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9753a1cbb2d54292aeac0d805e240cf0: No bootstrap required, opened a new log
10:27:51.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:51.603806 9142 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 570b75c366184198a09749ea161d487c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
10:27:51.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:51.604140 9142 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
10:27:51.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.610882 8966 raft_consensus.cc:348] T 00000000000000000000000000000000 P 9753a1cbb2d54292aeac0d805e240cf0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9753a1cbb2d54292aeac0d805e240cf0" member_type: VOTER last_known_addr { host: "127.8.173.190" port: 41013 } } peers { permanent_uuid: "5a5ddfff0bfc4646a4c8c88324092bbd" member_type: VOTER last_known_addr { host: "127.8.173.189" port: 41829 } } peers { permanent_uuid: "570b75c366184198a09749ea161d487c" member_type: VOTER last_known_addr { host: "127.8.173.188" port: 43959 } }
10:27:51.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.611529 8966 raft_consensus.cc:374] T 00000000000000000000000000000000 P 9753a1cbb2d54292aeac0d805e240cf0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
10:27:51.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.611829 8966 raft_consensus.cc:729] T 00000000000000000000000000000000 P 9753a1cbb2d54292aeac0d805e240cf0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9753a1cbb2d54292aeac0d805e240cf0, State: Initialized, Role: FOLLOWER
10:27:51.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.612687 8966 consensus_queue.cc:260] T 00000000000000000000000000000000 P 9753a1cbb2d54292aeac0d805e240cf0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9753a1cbb2d54292aeac0d805e240cf0" member_type: VOTER last_known_addr { host: "127.8.173.190" port: 41013 } } peers { permanent_uuid: "5a5ddfff0bfc4646a4c8c88324092bbd" member_type: VOTER last_known_addr { host: "127.8.173.189" port: 41829 } } peers { permanent_uuid: "570b75c366184198a09749ea161d487c" member_type: VOTER last_known_addr { host: "127.8.173.188" port: 43959 } }
10:27:51.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.620837 9144 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9753a1cbb2d54292aeac0d805e240cf0 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9753a1cbb2d54292aeac0d805e240cf0" member_type: VOTER last_known_addr { host: "127.8.173.190" port: 41013 } } peers { permanent_uuid: "5a5ddfff0bfc4646a4c8c88324092bbd" member_type: VOTER last_known_addr { host: "127.8.173.189" port: 41829 } } peers { permanent_uuid: "570b75c366184198a09749ea161d487c" member_type: VOTER last_known_addr { host: "127.8.173.188" port: 43959 } } }
10:27:51.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.622398 9144 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9753a1cbb2d54292aeac0d805e240cf0 [sys.catalog]: This master's current role is: FOLLOWER
10:27:51.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.623672 8966 sys_catalog.cc:564] T 00000000000000000000000000000000 P 9753a1cbb2d54292aeac0d805e240cf0 [sys.catalog]: configured and running, proceeding with master startup.
10:27:51.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.593223 9043 master_runner.cc:186] Error getting master registration for 127.8.173.190:41013: OK, instance_id { permanent_uuid: "9753a1cbb2d54292aeac0d805e240cf0" instance_seqno: 1790677667942688 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
10:27:51.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.627449 9043 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
10:27:51.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:51.627914 9074 tablet.cc:2369] T 00000000000000000000000000000000 P 570b75c366184198a09749ea161d487c: Can't schedule compaction. Clean time has not been advanced past its initial value.
10:27:51.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:51.652733 8920 tablet.cc:2369] T 00000000000000000000000000000000 P 9753a1cbb2d54292aeac0d805e240cf0: Can't schedule compaction. Clean time has not been advanced past its initial value.
10:27:51.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:51.655210 9155 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 9753a1cbb2d54292aeac0d805e240cf0: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
10:27:51.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:51.655759 9155 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
10:27:51.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.652856 8898 master_runner.cc:186] Error getting master registration for 127.8.173.189:41829: OK, instance_id { permanent_uuid: "5a5ddfff0bfc4646a4c8c88324092bbd" instance_seqno: 1790677669666670 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
10:27:51.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.683352 8898 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
10:27:51.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.950222 9130 raft_consensus.cc:482] T 00000000000000000000000000000000 P 570b75c366184198a09749ea161d487c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
10:27:51.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.950795 9130 raft_consensus.cc:504] T 00000000000000000000000000000000 P 570b75c366184198a09749ea161d487c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9753a1cbb2d54292aeac0d805e240cf0" member_type: VOTER last_known_addr { host: "127.8.173.190" port: 41013 } } peers { permanent_uuid: "5a5ddfff0bfc4646a4c8c88324092bbd" member_type: VOTER last_known_addr { host: "127.8.173.189" port: 41829 } } peers { permanent_uuid: "570b75c366184198a09749ea161d487c" member_type: VOTER last_known_addr { host: "127.8.173.188" port: 43959 } }
10:27:51.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.953313 9130 leader_election.cc:288] T 00000000000000000000000000000000 P 570b75c366184198a09749ea161d487c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 9753a1cbb2d54292aeac0d805e240cf0 (127.8.173.190:41013), 5a5ddfff0bfc4646a4c8c88324092bbd (127.8.173.189:41829)
10:27:51.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.954418 8941 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "570b75c366184198a09749ea161d487c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9753a1cbb2d54292aeac0d805e240cf0" is_pre_election: true
10:27:51.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.955286 8941 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 9753a1cbb2d54292aeac0d805e240cf0 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 570b75c366184198a09749ea161d487c in term 0.
10:27:51.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.956688 9070 leader_election.cc:302] T 00000000000000000000000000000000 P 570b75c366184198a09749ea161d487c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 570b75c366184198a09749ea161d487c, 9753a1cbb2d54292aeac0d805e240cf0; no voters:
10:27:51.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.957523 9130 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 570b75c366184198a09749ea161d487c [term 0 FOLLOWER]: Leader pre-election won for term 1
10:27:51.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.957844 9130 raft_consensus.cc:482] T 00000000000000000000000000000000 P 570b75c366184198a09749ea161d487c [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
10:27:51.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.958122 9130 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 570b75c366184198a09749ea161d487c [term 0 FOLLOWER]: Advancing to term 1
10:27:51.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.975903 9130 raft_consensus.cc:504] T 00000000000000000000000000000000 P 570b75c366184198a09749ea161d487c [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9753a1cbb2d54292aeac0d805e240cf0" member_type: VOTER last_known_addr { host: "127.8.173.190" port: 41013 } } peers { permanent_uuid: "5a5ddfff0bfc4646a4c8c88324092bbd" member_type: VOTER last_known_addr { host: "127.8.173.189" port: 41829 } } peers { permanent_uuid: "570b75c366184198a09749ea161d487c" member_type: VOTER last_known_addr { host: "127.8.173.188" port: 43959 } }
10:27:51.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.981449 9130 leader_election.cc:288] T 00000000000000000000000000000000 P 570b75c366184198a09749ea161d487c [CANDIDATE]: Term 1 election: Requested vote from peers 9753a1cbb2d54292aeac0d805e240cf0 (127.8.173.190:41013), 5a5ddfff0bfc4646a4c8c88324092bbd (127.8.173.189:41829)
10:27:51.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.984441 8941 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "570b75c366184198a09749ea161d487c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9753a1cbb2d54292aeac0d805e240cf0"
10:27:51.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.984977 8941 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 9753a1cbb2d54292aeac0d805e240cf0 [term 0 FOLLOWER]: Advancing to term 1
10:27:51.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.991453 8941 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 9753a1cbb2d54292aeac0d805e240cf0 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 570b75c366184198a09749ea161d487c in term 1.
10:27:51.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.992527 9070 leader_election.cc:302] T 00000000000000000000000000000000 P 570b75c366184198a09749ea161d487c [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 570b75c366184198a09749ea161d487c, 9753a1cbb2d54292aeac0d805e240cf0; no voters:
10:27:52.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:51.999220 9130 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 570b75c366184198a09749ea161d487c [term 1 FOLLOWER]: Leader election won for term 1
10:27:52.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:52.011294 9130 raft_consensus.cc:686] T 00000000000000000000000000000000 P 570b75c366184198a09749ea161d487c [term 1 LEADER]: Becoming Leader. State: Replica: 570b75c366184198a09749ea161d487c, State: Running, Role: LEADER
10:27:52.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:52.013533 9130 consensus_queue.cc:237] T 00000000000000000000000000000000 P 570b75c366184198a09749ea161d487c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9753a1cbb2d54292aeac0d805e240cf0" member_type: VOTER last_known_addr { host: "127.8.173.190" port: 41013 } } peers { permanent_uuid: "5a5ddfff0bfc4646a4c8c88324092bbd" member_type: VOTER last_known_addr { host: "127.8.173.189" port: 41829 } } peers { permanent_uuid: "570b75c366184198a09749ea161d487c" member_type: VOTER last_known_addr { host: "127.8.173.188" port: 43959 } }
10:27:52.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:52.033476 9157 sys_catalog.cc:455] T 00000000000000000000000000000000 P 570b75c366184198a09749ea161d487c [sys.catalog]: SysCatalogTable state changed. Reason: New leader 570b75c366184198a09749ea161d487c. Latest consensus state: current_term: 1 leader_uuid: "570b75c366184198a09749ea161d487c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9753a1cbb2d54292aeac0d805e240cf0" member_type: VOTER last_known_addr { host: "127.8.173.190" port: 41013 } } peers { permanent_uuid: "5a5ddfff0bfc4646a4c8c88324092bbd" member_type: VOTER last_known_addr { host: "127.8.173.189" port: 41829 } } peers { permanent_uuid: "570b75c366184198a09749ea161d487c" member_type: VOTER last_known_addr { host: "127.8.173.188" port: 43959 } } }
10:27:52.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:52.034195 9157 sys_catalog.cc:458] T 00000000000000000000000000000000 P 570b75c366184198a09749ea161d487c [sys.catalog]: This master's current role is: LEADER
10:27:52.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:52.037410 9159 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
10:27:52.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:52.039965 9041 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.173.188" port: 43959 } attempt: 7
10:27:52.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:52.048038 9159 catalog_manager.cc:1484] Initializing Kudu cluster ID...
10:27:52.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:52.076797 8941 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 9753a1cbb2d54292aeac0d805e240cf0 [term 1 FOLLOWER]: Refusing update from remote peer 570b75c366184198a09749ea161d487c: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
10:27:52.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:52.080240 9157 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 570b75c366184198a09749ea161d487c [LEADER]: Connected to new peer: Peer: permanent_uuid: "9753a1cbb2d54292aeac0d805e240cf0" member_type: VOTER last_known_addr { host: "127.8.173.190" port: 41013 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
10:27:52.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:52.086707 9041 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5a5ddfff0bfc4646a4c8c88324092bbd: Bootstrap starting.
10:27:52.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:52.093931 9041 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 5a5ddfff0bfc4646a4c8c88324092bbd: Neither blocks nor log segments found. Creating new log.
10:27:52.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:52.097266 9041 log.cc:824] T 00000000000000000000000000000000 P 5a5ddfff0bfc4646a4c8c88324092bbd: Log is configured to *not* fsync() on all Append() calls
10:27:52.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:52.112224 9041 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5a5ddfff0bfc4646a4c8c88324092bbd: No bootstrap required, opened a new log
10:27:52.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:52.127864 9144 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9753a1cbb2d54292aeac0d805e240cf0 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 570b75c366184198a09749ea161d487c. Latest consensus state: current_term: 1 leader_uuid: "570b75c366184198a09749ea161d487c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9753a1cbb2d54292aeac0d805e240cf0" member_type: VOTER last_known_addr { host: "127.8.173.190" port: 41013 } } peers { permanent_uuid: "5a5ddfff0bfc4646a4c8c88324092bbd" member_type: VOTER last_known_addr { host: "127.8.173.189" port: 41829 } } peers { permanent_uuid: "570b75c366184198a09749ea161d487c" member_type: VOTER last_known_addr { host: "127.8.173.188" port: 43959 } } }
10:27:52.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:52.128654 9144 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9753a1cbb2d54292aeac0d805e240cf0 [sys.catalog]: This master's current role is: FOLLOWER
10:27:52.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:52.140404 9161 mvcc.cc:205] Tried to move back new op lower bound from 7334615744804667392 to 7334615744593215488. Current Snapshot: MvccSnapshot[applied={T|T < 7334615744804667392}]
10:27:52.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:52.155826 9144 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9753a1cbb2d54292aeac0d805e240cf0 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "570b75c366184198a09749ea161d487c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9753a1cbb2d54292aeac0d805e240cf0" member_type: VOTER last_known_addr { host: "127.8.173.190" port: 41013 } } peers { permanent_uuid: "5a5ddfff0bfc4646a4c8c88324092bbd" member_type: VOTER last_known_addr { host: "127.8.173.189" port: 41829 } } peers { permanent_uuid: "570b75c366184198a09749ea161d487c" member_type: VOTER last_known_addr { host: "127.8.173.188" port: 43959 } } }
10:27:52.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:52.156957 9144 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9753a1cbb2d54292aeac0d805e240cf0 [sys.catalog]: This master's current role is: FOLLOWER
10:27:52.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:52.156579 9157 sys_catalog.cc:455] T 00000000000000000000000000000000 P 570b75c366184198a09749ea161d487c [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "570b75c366184198a09749ea161d487c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9753a1cbb2d54292aeac0d805e240cf0" member_type: VOTER last_known_addr { host: "127.8.173.190" port: 41013 } } peers { permanent_uuid: "5a5ddfff0bfc4646a4c8c88324092bbd" member_type: VOTER last_known_addr { host: "127.8.173.189" port: 41829 } } peers { permanent_uuid: "570b75c366184198a09749ea161d487c" member_type: VOTER last_known_addr { host: "127.8.173.188" port: 43959 } } }
10:27:52.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:52.157254 9157 sys_catalog.cc:458] T 00000000000000000000000000000000 P 570b75c366184198a09749ea161d487c [sys.catalog]: This master's current role is: LEADER
10:27:52.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:52.169710 9159 catalog_manager.cc:1347] Generated new cluster ID: eee22064d9124ad786d50a6a738dce9b
10:27:52.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:52.169975 9159 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
10:27:52.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:52.194020 9041 raft_consensus.cc:348] T 00000000000000000000000000000000 P 5a5ddfff0bfc4646a4c8c88324092bbd [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9753a1cbb2d54292aeac0d805e240cf0" member_type: VOTER last_known_addr { host: "127.8.173.190" port: 41013 } } peers { permanent_uuid: "5a5ddfff0bfc4646a4c8c88324092bbd" member_type: VOTER last_known_addr { host: "127.8.173.189" port: 41829 } } peers { permanent_uuid: "570b75c366184198a09749ea161d487c" member_type: VOTER last_known_addr { host: "127.8.173.188" port: 43959 } }
10:27:52.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:52.194844 9041 raft_consensus.cc:374] T 00000000000000000000000000000000 P 5a5ddfff0bfc4646a4c8c88324092bbd [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
10:27:52.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:52.195113 9041 raft_consensus.cc:729] T 00000000000000000000000000000000 P 5a5ddfff0bfc4646a4c8c88324092bbd [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5a5ddfff0bfc4646a4c8c88324092bbd, State: Initialized, Role: FOLLOWER
10:27:52.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:52.196048 9041 consensus_queue.cc:260] T 00000000000000000000000000000000 P 5a5ddfff0bfc4646a4c8c88324092bbd [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9753a1cbb2d54292aeac0d805e240cf0" member_type: VOTER last_known_addr { host: "127.8.173.190" port: 41013 } } peers { permanent_uuid: "5a5ddfff0bfc4646a4c8c88324092bbd" member_type: VOTER last_known_addr { host: "127.8.173.189" port: 41829 } } peers { permanent_uuid: "570b75c366184198a09749ea161d487c" member_type: VOTER last_known_addr { host: "127.8.173.188" port: 43959 } }
10:27:52.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:52.212682 9167 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5a5ddfff0bfc4646a4c8c88324092bbd [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9753a1cbb2d54292aeac0d805e240cf0" member_type: VOTER last_known_addr { host: "127.8.173.190" port: 41013 } } peers { permanent_uuid: "5a5ddfff0bfc4646a4c8c88324092bbd" member_type: VOTER last_known_addr { host: "127.8.173.189" port: 41829 } } peers { permanent_uuid: "570b75c366184198a09749ea161d487c" member_type: VOTER last_known_addr { host: "127.8.173.188" port: 43959 } } }
10:27:52.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:52.215041 9167 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5a5ddfff0bfc4646a4c8c88324092bbd [sys.catalog]: This master's current role is: FOLLOWER
10:27:52.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:52.222772 9041 sys_catalog.cc:564] T 00000000000000000000000000000000 P 5a5ddfff0bfc4646a4c8c88324092bbd [sys.catalog]: configured and running, proceeding with master startup.
10:27:52.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:52.222934 9016 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "570b75c366184198a09749ea161d487c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5a5ddfff0bfc4646a4c8c88324092bbd" is_pre_election: true
10:27:52.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:52.223819 9016 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 5a5ddfff0bfc4646a4c8c88324092bbd [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 570b75c366184198a09749ea161d487c in term 0.
10:27:52.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:52.225226 9014 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 5a5ddfff0bfc4646a4c8c88324092bbd [term 0 FOLLOWER]: Advancing to term 1
10:27:52.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:52.228116 9015 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "570b75c366184198a09749ea161d487c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5a5ddfff0bfc4646a4c8c88324092bbd"
10:27:52.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:52.239127 9014 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 5a5ddfff0bfc4646a4c8c88324092bbd [term 1 FOLLOWER]: Refusing update from remote peer 570b75c366184198a09749ea161d487c: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
10:27:52.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:52.249461 9159 catalog_manager.cc:1370] Generated new certificate authority record
10:27:52.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:52.252007 9159 catalog_manager.cc:1504] Loading token signing keys...
10:27:52.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:52.253685 9157 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 570b75c366184198a09749ea161d487c [LEADER]: Connected to new peer: Peer: permanent_uuid: "5a5ddfff0bfc4646a4c8c88324092bbd" member_type: VOTER last_known_addr { host: "127.8.173.189" port: 41829 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
10:27:52.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:52.285709 9159 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 570b75c366184198a09749ea161d487c: Generated new TSK 0
10:27:52.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:52.286772 9159 catalog_manager.cc:1514] Initializing in-progress tserver states...
10:27:52.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:52.323421 9167 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5a5ddfff0bfc4646a4c8c88324092bbd [sys.catalog]: SysCatalogTable state changed. Reason: New leader 570b75c366184198a09749ea161d487c. Latest consensus state: current_term: 1 leader_uuid: "570b75c366184198a09749ea161d487c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9753a1cbb2d54292aeac0d805e240cf0" member_type: VOTER last_known_addr { host: "127.8.173.190" port: 41013 } } peers { permanent_uuid: "5a5ddfff0bfc4646a4c8c88324092bbd" member_type: VOTER last_known_addr { host: "127.8.173.189" port: 41829 } } peers { permanent_uuid: "570b75c366184198a09749ea161d487c" member_type: VOTER last_known_addr { host: "127.8.173.188" port: 43959 } } }
10:27:52.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:52.324172 9167 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5a5ddfff0bfc4646a4c8c88324092bbd [sys.catalog]: This master's current role is: FOLLOWER
10:27:52.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:52.334652 9175 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5a5ddfff0bfc4646a4c8c88324092bbd [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "570b75c366184198a09749ea161d487c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9753a1cbb2d54292aeac0d805e240cf0" member_type: VOTER last_known_addr { host: "127.8.173.190" port: 41013 } } peers { permanent_uuid: "5a5ddfff0bfc4646a4c8c88324092bbd" member_type: VOTER last_known_addr { host: "127.8.173.189" port: 41829 } } peers { permanent_uuid: "570b75c366184198a09749ea161d487c" member_type: VOTER last_known_addr { host: "127.8.173.188" port: 43959 } } }
10:27:52.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:52.336580 9175 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5a5ddfff0bfc4646a4c8c88324092bbd [sys.catalog]: This master's current role is: FOLLOWER
10:27:52.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:52.337680 9130 sys_catalog.cc:455] T 00000000000000000000000000000000 P 570b75c366184198a09749ea161d487c [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "570b75c366184198a09749ea161d487c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9753a1cbb2d54292aeac0d805e240cf0" member_type: VOTER last_known_addr { host: "127.8.173.190" port: 41013 } } peers { permanent_uuid: "5a5ddfff0bfc4646a4c8c88324092bbd" member_type: VOTER last_known_addr { host: "127.8.173.189" port: 41829 } } peers { permanent_uuid: "570b75c366184198a09749ea161d487c" member_type: VOTER last_known_addr { host: "127.8.173.188" port: 43959 } } }
10:27:52.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:52.338336 9130 sys_catalog.cc:458] T 00000000000000000000000000000000 P 570b75c366184198a09749ea161d487c [sys.catalog]: This master's current role is: LEADER
10:27:52.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:52.339015 8886 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasks36zsq/build/tsan/bin/kudu
10:27:52.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasks36zsq/build/tsan/bin/kudu
10:27:52.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/ts-0/wal
10:27:52.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/ts-0/data
10:27:52.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:27:52.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:27:52.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:27:52.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:27:52.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:27:52.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:27:52.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/ts-0/logs
10:27:52.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/ts-0/data/info.pb
10:27:52.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:27:52.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:27:52.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:27:52.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:27:52.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:27:52.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:27:52.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:27:52.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:27:52.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
10:27:52.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:27:52.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.173.129:0
10:27:52.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.173.129
10:27:52.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.173.129
10:27:52.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:27:52.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.173.190:41013,127.8.173.189:41829,127.8.173.188:43959
10:27:52.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.173.148:33799
10:27:52.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:27:52.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:27:52.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
10:27:52.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:52.396955 9184 catalog_manager.cc:1259] Loaded cluster ID: eee22064d9124ad786d50a6a738dce9b
10:27:52.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:52.397209 9184 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 5a5ddfff0bfc4646a4c8c88324092bbd: loading cluster ID for follower catalog manager: success
10:27:52.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:52.405095 9184 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 5a5ddfff0bfc4646a4c8c88324092bbd: acquiring CA information for follower catalog manager: success
10:27:52.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:52.410362 9184 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 5a5ddfff0bfc4646a4c8c88324092bbd: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
10:27:52.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:52.661598 9155 catalog_manager.cc:1259] Loaded cluster ID: eee22064d9124ad786d50a6a738dce9b
10:27:52.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:52.662190 9155 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 9753a1cbb2d54292aeac0d805e240cf0: loading cluster ID for follower catalog manager: success
10:27:52.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:52.682371 9155 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 9753a1cbb2d54292aeac0d805e240cf0: acquiring CA information for follower catalog manager: success
10:27:52.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:52.688159 9155 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 9753a1cbb2d54292aeac0d805e240cf0: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
10:27:52.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:52.833180 9182 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
10:27:52.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:52.834141 9182 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:27:52.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:52.835029 9182 flags.cc:425] Enabled unsafe flag: --never_fsync=true
10:27:52.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:52.891518 9182 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
10:27:52.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:52.893303 9182 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.8.173.129
10:27:52.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:52.955181 9182 tablet_server_runner.cc:78] Tablet server non-default flags:
10:27:52.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:27:52.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.173.148:33799
10:27:52.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:27:52.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:27:52.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/ts-0/data
10:27:52.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/ts-0/wal
10:27:52.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:27:52.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:27:52.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.173.129:0
10:27:52.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:27:52.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:27:52.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:27:52.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/ts-0/data/info.pb
10:27:52.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.173.129
10:27:52.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:27:52.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.173.190:41013,127.8.173.189:41829,127.8.173.188:43959
10:27:52.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:27:52.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:27:52.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:27:52.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:27:52.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:27:52.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.173.129
10:27:52.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/ts-0/logs
10:27:52.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:27:52.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:27:52.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
10:27:52.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
10:27:52.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
10:27:52.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
10:27:52.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:27:52.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 09:27:27 UTC on ccycloud.quasar-stwief.root.comops.site
10:27:52.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679491
10:27:52.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
10:27:52.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:52.957507 9182 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:27:52.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:52.972872 9182 file_cache.cc:493] Constructed file cache file cache with capacity 419430
10:27:53.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:53.033630 9192 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:55.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:53.044071 9191 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:55.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:55.219573 9182 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
10:27:55.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:55.219549 9193 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2187 milliseconds
10:27:55.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:53.034906 9194 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:55.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:55.224093 9182 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:27:55.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:55.226732 9182 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:27:55.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:55.228171 9182 hybrid_clock.cc:648] HybridClock initialized: now 1790677675228136 us; error 39 us; skew 500 ppm
10:27:55.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:55.229190 9182 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:27:55.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:55.241138 9182 webserver.cc:460] Webserver started at http://127.8.173.129:40041/ using document root <none> and password file <none>
10:27:55.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:55.242305 9182 fs_manager.cc:359] Metadata directory not provided
10:27:55.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:55.242576 9182 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:27:55.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:55.243108 9182 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
10:27:55.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:55.250185 9182 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/ts-0/data/instance:
10:27:55.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3bd2dacdf5d14658bb99a8797b4206fd"
10:27:55.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 10:27:55 on dist-test-slave-9qlq"
10:27:55.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:55.251621 9182 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/ts-0/wal/instance:
10:27:55.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3bd2dacdf5d14658bb99a8797b4206fd"
10:27:55.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 10:27:55 on dist-test-slave-9qlq"
10:27:55.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:55.259718 9182 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.005s
10:27:55.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:55.266224 9201 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
10:27:55.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:55.267338 9182 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.006s sys 0.000s
10:27:55.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:55.267746 9182 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/ts-0/data,/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/ts-0/wal
10:27:55.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3bd2dacdf5d14658bb99a8797b4206fd"
10:27:55.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 10:27:55 on dist-test-slave-9qlq"
10:27:55.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:55.268124 9182 fs_report.cc:383] FS layout report
10:27:55.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:27:55.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/ts-0/wal
10:27:55.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/ts-0/wal
10:27:55.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/ts-0/data/data
10:27:55.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:27:55.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:27:55.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:27:55.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:27:55.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:27:55.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:27:55.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:27:55.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:27:55.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:27:55.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:27:55.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:27:55.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:55.311318 9182 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
10:27:55.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:55.313091 9182 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:27:55.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:55.313592 9182 kserver.cc:163] Server-wide thread pool size limit: 3276
10:27:55.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:55.316252 9182 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
10:27:55.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:55.321388 9182 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
10:27:55.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:55.321606 9182 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
10:27:55.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:55.321952 9182 ts_tablet_manager.cc:610] Registered 0 tablets
10:27:55.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:55.322139 9182 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
10:27:55.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:55.499428 9182 rpc_server.cc:307] RPC server started. Bound to: 127.8.173.129:40149
10:27:55.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:55.499511 9314 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.8.173.129:40149 every 8 connection(s)
10:27:55.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:55.502403 9182 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/ts-0/data/info.pb
10:27:55.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:55.510229 8886 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasks36zsq/build/tsan/bin/kudu as pid 9182
10:27:55.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:55.510785 8886 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/ts-0/wal/instance
10:27:55.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:55.522574 8886 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasks36zsq/build/tsan/bin/kudu
10:27:55.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasks36zsq/build/tsan/bin/kudu
10:27:55.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/ts-1/wal
10:27:55.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/ts-1/data
10:27:55.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:27:55.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:27:55.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:27:55.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:27:55.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:27:55.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:27:55.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/ts-1/logs
10:27:55.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/ts-1/data/info.pb
10:27:55.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:27:55.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:27:55.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:27:55.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:27:55.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:27:55.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:27:55.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:27:55.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:27:55.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
10:27:55.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:27:55.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.173.130:0
10:27:55.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.173.130
10:27:55.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.173.130
10:27:55.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:27:55.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.173.190:41013,127.8.173.189:41829,127.8.173.188:43959
10:27:55.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.173.148:33799
10:27:55.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:27:55.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:27:55.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
10:27:55.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:55.567481 9316 heartbeater.cc:347] Connected to a master server at 127.8.173.189:41829
10:27:55.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:55.567786 9315 heartbeater.cc:347] Connected to a master server at 127.8.173.188:43959
10:27:55.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:55.568012 9316 heartbeater.cc:464] Registering TS with master...
10:27:55.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:55.568353 9315 heartbeater.cc:464] Registering TS with master...
10:27:55.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:55.569262 9316 heartbeater.cc:511] Master 127.8.173.189:41829 requested a full tablet report, sending...
10:27:55.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:55.569422 9315 heartbeater.cc:511] Master 127.8.173.188:43959 requested a full tablet report, sending...
10:27:55.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:55.572250 9085 ts_manager.cc:194] Registered new tserver with Master: 3bd2dacdf5d14658bb99a8797b4206fd (127.8.173.129:40149)
10:27:55.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:55.573060 9006 ts_manager.cc:194] Registered new tserver with Master: 3bd2dacdf5d14658bb99a8797b4206fd (127.8.173.129:40149)
10:27:55.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:55.575290 9085 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.8.173.129:39333
10:27:55.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:55.578740 9317 heartbeater.cc:347] Connected to a master server at 127.8.173.190:41013
10:27:55.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:55.579030 9317 heartbeater.cc:464] Registering TS with master...
10:27:55.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:55.579797 9317 heartbeater.cc:511] Master 127.8.173.190:41013 requested a full tablet report, sending...
10:27:55.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:55.582175 8931 ts_manager.cc:194] Registered new tserver with Master: 3bd2dacdf5d14658bb99a8797b4206fd (127.8.173.129:40149)
10:27:56.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:56.001083 9322 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
10:27:56.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:56.001641 9322 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:27:56.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:56.002065 9322 flags.cc:425] Enabled unsafe flag: --never_fsync=true
10:27:56.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:56.034361 9322 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
10:27:56.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:56.035166 9322 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.8.173.130
10:27:56.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:56.071522 9322 tablet_server_runner.cc:78] Tablet server non-default flags:
10:27:56.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:27:56.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.173.148:33799
10:27:56.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:27:56.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:27:56.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/ts-1/data
10:27:56.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/ts-1/wal
10:27:56.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:27:56.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:27:56.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.173.130:0
10:27:56.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:27:56.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:27:56.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:27:56.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/ts-1/data/info.pb
10:27:56.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.173.130
10:27:56.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:27:56.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.173.190:41013,127.8.173.189:41829,127.8.173.188:43959
10:27:56.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:27:56.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:27:56.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:27:56.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:27:56.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:27:56.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.173.130
10:27:56.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/ts-1/logs
10:27:56.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:27:56.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:27:56.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
10:27:56.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
10:27:56.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
10:27:56.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
10:27:56.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:27:56.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 09:27:27 UTC on ccycloud.quasar-stwief.root.comops.site
10:27:56.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679491
10:27:56.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
10:27:56.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:56.072798 9322 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:27:56.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:56.074443 9322 file_cache.cc:493] Constructed file cache file cache with capacity 419430
10:27:56.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:56.105785 9335 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:56.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:56.580122 9315 heartbeater.cc:503] Master 127.8.173.188:43959 was elected leader, sending a full tablet report...
10:27:56.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:56.850189 9311 debug-util.cc:398] Leaking SignalData structure 0x7b08000af060 after lost signal to thread 9186
10:27:56.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:56.851064 9311 debug-util.cc:398] Leaking SignalData structure 0x7b08000aff60 after lost signal to thread 9314
10:27:57.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:56.106099 9332 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:57.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:56.106734 9333 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:57.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:56.107367 9322 server_base.cc:1034] running on GCE node
10:27:57.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:57.293661 9322 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:27:57.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:57.296293 9322 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:27:57.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:57.297737 9322 hybrid_clock.cc:648] HybridClock initialized: now 1790677677297676 us; error 73 us; skew 500 ppm
10:27:57.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:57.298491 9322 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:27:57.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:57.306021 9322 webserver.cc:460] Webserver started at http://127.8.173.130:37097/ using document root <none> and password file <none>
10:27:57.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:57.306854 9322 fs_manager.cc:359] Metadata directory not provided
10:27:57.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:57.307045 9322 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:27:57.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:57.307416 9322 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
10:27:57.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:57.312156 9322 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/ts-1/data/instance:
10:27:57.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "931e57bca4ea4a1da05d1121c8149125"
10:27:57.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 10:27:57 on dist-test-slave-9qlq"
10:27:57.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:57.313228 9322 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/ts-1/wal/instance:
10:27:57.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "931e57bca4ea4a1da05d1121c8149125"
10:27:57.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 10:27:57 on dist-test-slave-9qlq"
10:27:57.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:57.320268 9322 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
10:27:57.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:57.326336 9344 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
10:27:57.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:57.327387 9322 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.006s sys 0.000s
10:27:57.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:57.327760 9322 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/ts-1/data,/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/ts-1/wal
10:27:57.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "931e57bca4ea4a1da05d1121c8149125"
10:27:57.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 10:27:57 on dist-test-slave-9qlq"
10:27:57.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:57.328104 9322 fs_report.cc:383] FS layout report
10:27:57.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:27:57.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/ts-1/wal
10:27:57.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/ts-1/wal
10:27:57.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/ts-1/data/data
10:27:57.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:27:57.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:27:57.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:27:57.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:27:57.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:27:57.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:27:57.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:27:57.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:27:57.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:27:57.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:27:57.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:27:57.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:57.366093 9322 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
10:27:57.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:57.368206 9322 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:27:57.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:57.368902 9322 kserver.cc:163] Server-wide thread pool size limit: 3276
10:27:57.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:57.372617 9322 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
10:27:57.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:57.377172 9322 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
10:27:57.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:57.377351 9322 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
10:27:57.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:57.377575 9322 ts_tablet_manager.cc:610] Registered 0 tablets
10:27:57.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:57.377704 9322 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
10:27:57.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:57.562934 9322 rpc_server.cc:307] RPC server started. Bound to: 127.8.173.130:39729
10:27:57.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:57.563019 9456 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.8.173.130:39729 every 8 connection(s)
10:27:57.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:57.565908 9322 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/ts-1/data/info.pb
10:27:57.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:57.567500 8886 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasks36zsq/build/tsan/bin/kudu as pid 9322
10:27:57.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:57.568141 8886 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/ts-1/wal/instance
10:27:57.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:57.607088 8886 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasks36zsq/build/tsan/bin/kudu
10:27:57.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasks36zsq/build/tsan/bin/kudu
10:27:57.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/ts-2/wal
10:27:57.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/ts-2/data
10:27:57.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:27:57.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:27:57.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:27:57.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:27:57.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:27:57.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:27:57.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/ts-2/logs
10:27:57.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/ts-2/data/info.pb
10:27:57.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:27:57.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:27:57.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:27:57.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:27:57.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:27:57.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:27:57.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:27:57.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:27:57.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
10:27:57.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:27:57.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.173.131:0
10:27:57.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.173.131
10:27:57.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.173.131
10:27:57.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:27:57.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.173.190:41013,127.8.173.189:41829,127.8.173.188:43959
10:27:57.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.173.148:33799
10:27:57.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:27:57.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:27:57.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
10:27:57.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:57.626097 9458 heartbeater.cc:347] Connected to a master server at 127.8.173.189:41829
10:27:57.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:57.626575 9458 heartbeater.cc:464] Registering TS with master...
10:27:57.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:57.626777 9457 heartbeater.cc:347] Connected to a master server at 127.8.173.188:43959
10:27:57.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:57.627127 9457 heartbeater.cc:464] Registering TS with master...
10:27:57.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:57.627779 9458 heartbeater.cc:511] Master 127.8.173.189:41829 requested a full tablet report, sending...
10:27:57.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:57.628054 9457 heartbeater.cc:511] Master 127.8.173.188:43959 requested a full tablet report, sending...
10:27:57.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:57.630424 9085 ts_manager.cc:194] Registered new tserver with Master: 931e57bca4ea4a1da05d1121c8149125 (127.8.173.130:39729)
10:27:57.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:57.632630 9085 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.8.173.130:37475
10:27:57.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:57.633430 9006 ts_manager.cc:194] Registered new tserver with Master: 931e57bca4ea4a1da05d1121c8149125 (127.8.173.130:39729)
10:27:57.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:57.640755 9459 heartbeater.cc:347] Connected to a master server at 127.8.173.190:41013
10:27:57.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:57.641109 9459 heartbeater.cc:464] Registering TS with master...
10:27:57.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:57.641852 9459 heartbeater.cc:511] Master 127.8.173.190:41013 requested a full tablet report, sending...
10:27:57.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:57.644693 8931 ts_manager.cc:194] Registered new tserver with Master: 931e57bca4ea4a1da05d1121c8149125 (127.8.173.130:39729)
10:27:57.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:57.956465 9467 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
10:27:57.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:57.956914 9467 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:27:57.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:57.957415 9467 flags.cc:425] Enabled unsafe flag: --never_fsync=true
10:27:58.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:58.007676 9467 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
10:27:58.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:58.008567 9467 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.8.173.131
10:27:58.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:58.043208 9467 tablet_server_runner.cc:78] Tablet server non-default flags:
10:27:58.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:27:58.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.173.148:33799
10:27:58.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:27:58.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:27:58.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/ts-2/data
10:27:58.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/ts-2/wal
10:27:58.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:27:58.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:27:58.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.173.131:0
10:27:58.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:27:58.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:27:58.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:27:58.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/ts-2/data/info.pb
10:27:58.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.173.131
10:27:58.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:27:58.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.173.190:41013,127.8.173.189:41829,127.8.173.188:43959
10:27:58.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:27:58.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:27:58.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:27:58.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:27:58.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:27:58.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.173.131
10:27:58.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/ts-2/logs
10:27:58.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:27:58.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:27:58.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
10:27:58.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
10:27:58.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
10:27:58.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
10:27:58.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:27:58.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 09:27:27 UTC on ccycloud.quasar-stwief.root.comops.site
10:27:58.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679491
10:27:58.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
10:27:58.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:58.044446 9467 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:27:58.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:58.046308 9467 file_cache.cc:493] Constructed file cache file cache with capacity 419430
10:27:58.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:58.069540 9474 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:58.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:58.639245 9457 heartbeater.cc:503] Master 127.8.173.188:43959 was elected leader, sending a full tablet report...
10:27:59.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:58.069551 9473 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:59.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:27:58.073103 9476 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:59.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:58.071473 9467 server_base.cc:1034] running on GCE node
10:27:59.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:59.295917 9467 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:27:59.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:59.297993 9467 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:27:59.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:59.299304 9467 hybrid_clock.cc:648] HybridClock initialized: now 1790677679299266 us; error 49 us; skew 500 ppm
10:27:59.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:59.300045 9467 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:27:59.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:59.306375 9467 webserver.cc:460] Webserver started at http://127.8.173.131:37801/ using document root <none> and password file <none>
10:27:59.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:59.307228 9467 fs_manager.cc:359] Metadata directory not provided
10:27:59.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:59.307415 9467 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:27:59.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:59.307812 9467 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
10:27:59.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:59.313011 9467 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/ts-2/data/instance:
10:27:59.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6979ff2dcdef4c05ad6b646e3077ab4c"
10:27:59.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 10:27:59 on dist-test-slave-9qlq"
10:27:59.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:59.314016 9467 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/ts-2/wal/instance:
10:27:59.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6979ff2dcdef4c05ad6b646e3077ab4c"
10:27:59.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 10:27:59 on dist-test-slave-9qlq"
10:27:59.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:59.320609 9467 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.007s sys 0.000s
10:27:59.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:59.325953 9484 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
10:27:59.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:59.326802 9467 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.005s sys 0.000s
10:27:59.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:59.327138 9467 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/ts-2/data,/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/ts-2/wal
10:27:59.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6979ff2dcdef4c05ad6b646e3077ab4c"
10:27:59.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 10:27:59 on dist-test-slave-9qlq"
10:27:59.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:59.327474 9467 fs_report.cc:383] FS layout report
10:27:59.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:27:59.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/ts-2/wal
10:27:59.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/ts-2/wal
10:27:59.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/ts-2/data/data
10:27:59.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:27:59.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:27:59.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:27:59.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:27:59.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:27:59.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:27:59.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:27:59.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:27:59.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:27:59.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:27:59.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:27:59.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:59.383677 9467 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
10:27:59.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:59.385093 9467 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:27:59.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:59.385506 9467 kserver.cc:163] Server-wide thread pool size limit: 3276
10:27:59.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:59.387861 9467 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
10:27:59.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:59.391857 9467 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
10:27:59.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:59.392045 9467 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
10:27:59.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:59.392328 9467 ts_tablet_manager.cc:610] Registered 0 tablets
10:27:59.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:59.392537 9467 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.001s sys 0.000s
10:27:59.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:59.544231 9467 rpc_server.cc:307] RPC server started. Bound to: 127.8.173.131:44175
10:27:59.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:59.544373 9596 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.8.173.131:44175 every 8 connection(s)
10:27:59.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:59.547065 9467 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/ts-2/data/info.pb
10:27:59.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:59.557175 8886 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasks36zsq/build/tsan/bin/kudu as pid 9467
10:27:59.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:59.557720 8886 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster1083802458298285634/ts-2/wal/instance
10:27:59.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:59.611027 9597 heartbeater.cc:347] Connected to a master server at 127.8.173.188:43959
10:27:59.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:59.611274 9598 heartbeater.cc:347] Connected to a master server at 127.8.173.189:41829
10:27:59.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:59.611514 9597 heartbeater.cc:464] Registering TS with master...
10:27:59.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:59.611578 9598 heartbeater.cc:464] Registering TS with master...
10:27:59.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:59.612722 9598 heartbeater.cc:511] Master 127.8.173.189:41829 requested a full tablet report, sending...
10:27:59.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:59.612825 9597 heartbeater.cc:511] Master 127.8.173.188:43959 requested a full tablet report, sending...
10:27:59.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:59.614992 9085 ts_manager.cc:194] Registered new tserver with Master: 6979ff2dcdef4c05ad6b646e3077ab4c (127.8.173.131:44175)
10:27:59.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:59.615502 9006 ts_manager.cc:194] Registered new tserver with Master: 6979ff2dcdef4c05ad6b646e3077ab4c (127.8.173.131:44175)
10:27:59.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:59.616613 9085 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.8.173.131:37399
10:27:59.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:59.619778 9599 heartbeater.cc:347] Connected to a master server at 127.8.173.190:41013
10:27:59.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:59.620010 9599 heartbeater.cc:464] Registering TS with master...
10:27:59.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:59.620555 9599 heartbeater.cc:511] Master 127.8.173.190:41013 requested a full tablet report, sending...
10:27:59.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:59.622888 8930 ts_manager.cc:194] Registered new tserver with Master: 6979ff2dcdef4c05ad6b646e3077ab4c (127.8.173.131:44175)
10:27:59.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:27:59.626740 8886 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
10:27:59.637 [DEBUG - main] (MiniKuduCluster.java:195) Response:
10:27:59.639 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
10:27:59.650 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.8.173.190"
port: 41013
}
bound_http_address {
host: "127.8.173.190"
port: 41023
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.8.173.189"
port: 41829
}
bound_http_address {
host: "127.8.173.189"
port: 38431
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.8.173.188"
port: 43959
}
bound_http_address {
host: "127.8.173.188"
port: 37265
}
}
}
10:27:59.656 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
10:27:59.659 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.8.173.129"
port: 40149
}
bound_http_address {
host: "127.8.173.129"
port: 40041
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.8.173.130"
port: 39729
}
bound_http_address {
host: "127.8.173.130"
port: 37097
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.8.173.131"
port: 44175
}
bound_http_address {
host: "127.8.173.131"
port: 37801
}
}
}
10:27:59.659 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
10:27:59.848 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
10:27:59.950 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-tasks36zsq/test-tmp
10:27:59.951 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
10:27:59.951 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-tasks36zsq/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
10:27:59.954 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
10:28:00.303 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
10:28:00.304 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
10:28:00.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:00.307195 9608 mini_oidc.cc:152] Starting JWKS server
10:28:00.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:00.310698 9608 webserver.cc:460] Webserver started at http://127.0.0.1:37887/ using document root <none> and password file <none>
10:28:00.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:00.311085 9608 mini_oidc.cc:189] Starting OIDC Discovery server
10:28:00.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:00.312708 9608 webserver.cc:460] Webserver started at http://127.0.0.1:42231/ using document root <none> and password file <none>
10:28:00.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
10:28:00.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/krb5kdc/principal' for realm 'KRBTEST.COM',
10:28:00.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
10:28:00.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 10:28:00 dist-test-slave-9qlq krb5kdc[9621](info): setting up network...
10:28:00.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
10:28:00.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 10:28:00 dist-test-slave-9qlq krb5kdc[9621](info): set up 2 sockets
10:28:00.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 10:28:00 dist-test-slave-9qlq krb5kdc[9621](info): commencing operation
10:28:00.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
10:28:00.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:00.620484 9597 heartbeater.cc:503] Master 127.8.173.188:43959 was elected leader, sending a full tablet report...
10:28:01.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:01.421423 9608 mini_kdc.cc:121] Time spent starting KDC: real 1.085s user 0.001s sys 0.018s
10:28:01.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
10:28:01.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
10:28:01.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
10:28:01.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
10:28:01.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
10:28:01.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
10:28:01.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
10:28:01.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
10:28:01.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
10:28:01.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
10:28:01.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/krb5kdc/test-user.keytab.
10:28:01.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/krb5kdc/test-user.keytab.
10:28:01.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 10:28:01 dist-test-slave-9qlq krb5kdc[9621](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790677681, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
10:28:01.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
10:28:01.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T10:28:01Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
10:28:01.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T10:28:01Z Disabled control of system clock
10:28:01.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.9.98.62@KRBTEST.COM; defaulting to no policy
10:28:01.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
10:28:01.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.9.98.62@KRBTEST.COM" created.
10:28:01.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
10:28:01.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.9.98.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/krb5kdc/kudu_127.9.98.62.keytab.
10:28:01.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.9.98.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/krb5kdc/kudu_127.9.98.62.keytab.
10:28:01.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:01.609787 9608 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasks36zsq/build/tsan/bin/kudu
10:28:01.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasks36zsq/build/tsan/bin/kudu
10:28:01.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-0/wal
10:28:01.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-0/data
10:28:01.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:28:01.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:28:01.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:28:01.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:28:01.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:28:01.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:28:01.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-0/logs
10:28:01.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-0/data/info.pb
10:28:01.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:28:01.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:28:01.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:28:01.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:28:01.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:28:01.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:28:01.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:28:01.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:28:01.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
10:28:01.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:28:01.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:28:01.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:28:01.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.98.62:37111
10:28:01.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.98.62
10:28:01.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:28:01.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/krb5kdc/kudu_127.9.98.62.keytab
10:28:01.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.98.62
10:28:01.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
10:28:01.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
10:28:01.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
10:28:01.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.98.20:38125
10:28:01.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:28:01.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:28:01.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:28:01.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:28:01.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.98.62:37111,127.9.98.61:42629,127.9.98.60:43787
10:28:01.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:28:01.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37887/jwks.json
10:28:01.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
10:28:01.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:01.910293 9634 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
10:28:01.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:01.910858 9634 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:28:01.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:01.911423 9634 flags.cc:425] Enabled unsafe flag: --never_fsync=true
10:28:01.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:01.941407 9634 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
10:28:01.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:01.941681 9634 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
10:28:01.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:01.942109 9634 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
10:28:01.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:01.942405 9634 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
10:28:01.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:01.942718 9634 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
10:28:01.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:01.943080 9634 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
10:28:01.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:01.943310 9634 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37887/jwks.json
10:28:01.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:01.978375 9634 master_runner.cc:386] Master server non-default flags:
10:28:01.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:28:01.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.98.20:38125
10:28:01.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:28:01.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:28:01.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-0/data
10:28:01.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-0/wal
10:28:01.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
10:28:01.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:28:01.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.98.62:37111,127.9.98.61:42629,127.9.98.60:43787
10:28:01.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/krb5kdc/kudu_127.9.98.62.keytab
10:28:01.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.98.62
10:28:01.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:28:01.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:28:01.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:28:01.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.98.62:37111
10:28:01.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:28:01.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:28:01.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:28:01.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37887/jwks.json
10:28:01.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
10:28:01.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
10:28:01.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
10:28:01.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:28:01.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:28:01.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-0/data/info.pb
10:28:01.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.98.62
10:28:01.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:28:01.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:28:01.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:28:01.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:28:01.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:28:01.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:28:01.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-0/logs
10:28:01.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:28:01.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:28:01.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
10:28:01.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
10:28:01.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
10:28:01.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
10:28:01.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:28:01.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 09:27:27 UTC on ccycloud.quasar-stwief.root.comops.site
10:28:01.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679491
10:28:01.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
10:28:01.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:01.979815 9634 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:28:01.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:01.981082 9634 file_cache.cc:493] Constructed file cache file cache with capacity 419430
10:28:02.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:01.999990 9643 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:28:03.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:01.999994 9641 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:28:03.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:02.000595 9640 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:28:03.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:02.001637 9634 server_base.cc:1034] running on GCE node
10:28:03.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:03.216073 9634 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:28:03.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:03.219178 9634 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:28:03.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:03.220609 9634 hybrid_clock.cc:648] HybridClock initialized: now 1790677683220567 us; error 59 us; skew 500 ppm
10:28:03.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:03.221383 9634 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:28:03.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 10:28:03 dist-test-slave-9qlq krb5kdc[9621](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790677683, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.98.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
10:28:03.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:03.227494 9634 init.cc:377] Logged in from keytab as kudu/127.9.98.62@KRBTEST.COM (short username kudu)
10:28:03.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:03.232975 9634 webserver.cc:460] Webserver started at http://127.9.98.62:41453/ using document root <none> and password file <none>
10:28:03.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:03.233876 9634 fs_manager.cc:359] Metadata directory not provided
10:28:03.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:03.234047 9634 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:28:03.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:03.234553 9634 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
10:28:03.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:03.239012 9634 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-0/data/instance:
10:28:03.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d6be5e30a64a408f85d5b2b07a3cca91"
10:28:03.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 10:28:03 on dist-test-slave-9qlq"
10:28:03.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:03.239982 9634 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-0/wal/instance:
10:28:03.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d6be5e30a64a408f85d5b2b07a3cca91"
10:28:03.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 10:28:03 on dist-test-slave-9qlq"
10:28:03.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:03.247428 9634 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.001s
10:28:03.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:03.253079 9652 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
10:28:03.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:03.253969 9634 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
10:28:03.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:03.254227 9634 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-0/data,/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-0/wal
10:28:03.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d6be5e30a64a408f85d5b2b07a3cca91"
10:28:03.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 10:28:03 on dist-test-slave-9qlq"
10:28:03.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:03.254510 9634 fs_report.cc:383] FS layout report
10:28:03.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:28:03.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-0/wal
10:28:03.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-0/wal
10:28:03.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-0/data/data
10:28:03.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:28:03.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:28:03.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:28:03.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:28:03.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:28:03.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:28:03.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:28:03.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:28:03.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:28:03.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:28:03.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:28:03.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:03.295246 9634 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
10:28:03.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:03.300709 9634 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:28:03.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:03.301096 9634 kserver.cc:163] Server-wide thread pool size limit: 3276
10:28:03.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:03.379781 9634 rpc_server.cc:307] RPC server started. Bound to: 127.9.98.62:37111
10:28:03.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:03.379851 9703 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.9.98.62:37111 every 8 connection(s)
10:28:03.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:03.383044 9634 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-0/data/info.pb
10:28:03.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:03.388737 9704 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
10:28:03.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:03.392457 9608 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasks36zsq/build/tsan/bin/kudu as pid 9634
10:28:03.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:03.392984 9608 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-0/wal/instance
10:28:03.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:03.404711 9704 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.98.62" port: 37111 } has no permanent_uuid. Determining permanent_uuid...
10:28:03.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.9.98.61@KRBTEST.COM; defaulting to no policy
10:28:03.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
10:28:03.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.9.98.61@KRBTEST.COM" created.
10:28:03.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 10:28:03 dist-test-slave-9qlq krb5kdc[9621](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790677683, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.98.62@KRBTEST.COM for kudu/127.9.98.62@KRBTEST.COM
10:28:03.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
10:28:03.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.9.98.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/krb5kdc/kudu_127.9.98.61.keytab.
10:28:03.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.9.98.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/krb5kdc/kudu_127.9.98.61.keytab.
10:28:03.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:03.440891 9608 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasks36zsq/build/tsan/bin/kudu
10:28:03.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasks36zsq/build/tsan/bin/kudu
10:28:03.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-1/wal
10:28:03.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-1/data
10:28:03.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:28:03.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:28:03.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:28:03.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:28:03.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:28:03.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:28:03.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-1/logs
10:28:03.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-1/data/info.pb
10:28:03.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:28:03.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:28:03.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:28:03.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:28:03.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:28:03.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:28:03.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:28:03.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:28:03.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
10:28:03.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:28:03.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:28:03.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:28:03.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.98.61:42629
10:28:03.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.98.61
10:28:03.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:28:03.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/krb5kdc/kudu_127.9.98.61.keytab
10:28:03.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.98.61
10:28:03.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
10:28:03.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
10:28:03.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
10:28:03.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.98.20:38125
10:28:03.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:28:03.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:28:03.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:28:03.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:28:03.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.98.62:37111,127.9.98.61:42629,127.9.98.60:43787
10:28:03.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:28:03.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37887/jwks.json
10:28:03.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
10:28:03.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:03.445593 9704 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.98.61" port: 42629 } has no permanent_uuid. Determining permanent_uuid...
10:28:03.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:03.448211 9656 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.98.61:42629: connect: Connection refused (error 111)
10:28:03.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:03.452309 9704 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.98.61:42629: Network error: Client connection negotiation failed: client connection to 127.9.98.61:42629: connect: Connection refused (error 111)
10:28:03.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:03.501861 9704 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.98.61" port: 42629 } attempt: 1
10:28:03.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:03.505386 9704 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.98.61:42629: Network error: Client connection negotiation failed: client connection to 127.9.98.61:42629: connect: Connection refused (error 111)
10:28:03.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:03.573765 9704 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.98.61" port: 42629 } attempt: 2
10:28:03.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:03.581243 9704 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.98.61:42629: Network error: Client connection negotiation failed: client connection to 127.9.98.61:42629: connect: Connection refused (error 111)
10:28:03.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:03.672641 9704 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.98.61" port: 42629 } attempt: 3
10:28:03.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:03.676391 9704 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.98.61:42629: Network error: Client connection negotiation failed: client connection to 127.9.98.61:42629: connect: Connection refused (error 111)
10:28:03.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:03.780584 9710 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
10:28:03.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:03.781104 9710 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:28:03.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:03.781646 9710 flags.cc:425] Enabled unsafe flag: --never_fsync=true
10:28:03.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:03.818295 9710 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
10:28:03.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:03.818486 9710 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
10:28:03.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:03.818711 9710 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
10:28:03.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:03.818883 9710 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
10:28:03.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:03.819058 9710 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
10:28:03.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:03.819222 9710 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
10:28:03.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:03.819345 9710 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37887/jwks.json
10:28:03.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:03.819790 9704 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.98.61" port: 42629 } attempt: 4
10:28:03.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:03.823362 9704 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.98.61:42629: Network error: Client connection negotiation failed: client connection to 127.9.98.61:42629: connect: Connection refused (error 111)
10:28:03.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:03.855515 9710 master_runner.cc:386] Master server non-default flags:
10:28:03.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:28:03.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.98.20:38125
10:28:03.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:28:03.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:28:03.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-1/data
10:28:03.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-1/wal
10:28:03.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
10:28:03.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:28:03.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.98.62:37111,127.9.98.61:42629,127.9.98.60:43787
10:28:03.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/krb5kdc/kudu_127.9.98.61.keytab
10:28:03.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.98.61
10:28:03.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:28:03.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:28:03.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:28:03.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.98.61:42629
10:28:03.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:28:03.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:28:03.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:28:03.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37887/jwks.json
10:28:03.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
10:28:03.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
10:28:03.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
10:28:03.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:28:03.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:28:03.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-1/data/info.pb
10:28:03.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.98.61
10:28:03.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:28:03.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:28:03.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:28:03.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:28:03.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:28:03.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:28:03.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-1/logs
10:28:03.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:28:03.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:28:03.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
10:28:03.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
10:28:03.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
10:28:03.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
10:28:03.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:28:03.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 09:27:27 UTC on ccycloud.quasar-stwief.root.comops.site
10:28:03.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679491
10:28:03.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
10:28:03.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:03.856632 9710 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:28:03.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:03.857625 9710 file_cache.cc:493] Constructed file cache file cache with capacity 419430
10:28:03.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:03.876266 9717 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:28:04.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:04.122820 9704 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.98.61" port: 42629 } attempt: 5
10:28:04.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:04.129259 9704 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.98.61:42629: Network error: Client connection negotiation failed: client connection to 127.9.98.61:42629: connect: Connection refused (error 111)
10:28:04.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:04.573765 9116 debug-util.cc:398] Leaking SignalData structure 0x7b08000bdc80 after lost signal to thread 9048
10:28:04.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:04.574445 9116 debug-util.cc:398] Leaking SignalData structure 0x7b08000b93e0 after lost signal to thread 9119
10:28:04.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:04.676807 9704 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.98.61" port: 42629 } attempt: 6
10:28:04.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:04.736605 9704 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.98.61:42629: Network error: Client connection negotiation failed: client connection to 127.9.98.61:42629: connect: Connection refused (error 111)
10:28:05.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:03.876436 9720 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:28:05.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:03.876664 9718 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:28:05.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:05.499899 9719 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1625 milliseconds
10:28:05.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:05.502130 9710 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
10:28:05.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:05.507879 9710 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:28:05.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:05.512847 9710 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:28:05.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:05.514452 9710 hybrid_clock.cc:648] HybridClock initialized: now 1790677685514367 us; error 74 us; skew 500 ppm
10:28:05.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:05.515254 9710 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:28:05.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 10:28:05 dist-test-slave-9qlq krb5kdc[9621](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790677685, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.98.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
10:28:05.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:05.524546 9710 init.cc:377] Logged in from keytab as kudu/127.9.98.61@KRBTEST.COM (short username kudu)
10:28:05.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:05.530660 9710 webserver.cc:460] Webserver started at http://127.9.98.61:34199/ using document root <none> and password file <none>
10:28:05.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:05.531580 9710 fs_manager.cc:359] Metadata directory not provided
10:28:05.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:05.531754 9710 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:28:05.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:05.532135 9710 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
10:28:05.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:05.537242 9710 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-1/data/instance:
10:28:05.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b3a4650bfa6e4be98ec505cb9046e767"
10:28:05.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 10:28:05 on dist-test-slave-9qlq"
10:28:05.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:05.538350 9710 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-1/wal/instance:
10:28:05.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b3a4650bfa6e4be98ec505cb9046e767"
10:28:05.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 10:28:05 on dist-test-slave-9qlq"
10:28:05.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:05.546501 9710 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.004s
10:28:05.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:05.553416 9730 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
10:28:05.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:05.554652 9710 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.002s sys 0.003s
10:28:05.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:05.554989 9710 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-1/data,/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-1/wal
10:28:05.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b3a4650bfa6e4be98ec505cb9046e767"
10:28:05.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 10:28:05 on dist-test-slave-9qlq"
10:28:05.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:05.555335 9710 fs_report.cc:383] FS layout report
10:28:05.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:28:05.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-1/wal
10:28:05.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-1/wal
10:28:05.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-1/data/data
10:28:05.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:28:05.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:28:05.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:28:05.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:28:05.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:28:05.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:28:05.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:28:05.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:28:05.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:28:05.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:28:05.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:28:05.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:05.608352 9710 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
10:28:05.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:05.615978 9710 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:28:05.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:05.616596 9710 kserver.cc:163] Server-wide thread pool size limit: 3276
10:28:05.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:05.743428 9710 rpc_server.cc:307] RPC server started. Bound to: 127.9.98.61:42629
10:28:05.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:05.743463 9781 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.9.98.61:42629 every 8 connection(s)
10:28:05.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:05.747679 9710 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-1/data/info.pb
10:28:05.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:05.755473 9782 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
10:28:05.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:05.757869 9608 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasks36zsq/build/tsan/bin/kudu as pid 9710
10:28:05.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:05.758350 9608 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-1/wal/instance
10:28:05.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:05.768951 9782 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.98.62" port: 37111 } has no permanent_uuid. Determining permanent_uuid...
10:28:05.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.9.98.60@KRBTEST.COM; defaulting to no policy
10:28:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:05.797068 9704 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.98.61" port: 42629 } attempt: 7
10:28:05.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
10:28:05.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.9.98.60@KRBTEST.COM" created.
10:28:05.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 10:28:05 dist-test-slave-9qlq krb5kdc[9621](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790677685, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.98.61@KRBTEST.COM for kudu/127.9.98.62@KRBTEST.COM
10:28:05.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 10:28:05 dist-test-slave-9qlq krb5kdc[9621](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790677683, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.98.62@KRBTEST.COM for kudu/127.9.98.61@KRBTEST.COM
10:28:05.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:05.940088 9782 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.98.61" port: 42629 } has no permanent_uuid. Determining permanent_uuid...
10:28:05.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:05.939976 9704 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.98.60" port: 43787 } has no permanent_uuid. Determining permanent_uuid...
10:28:05.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:05.946964 9704 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.98.60:43787: Network error: Client connection negotiation failed: client connection to 127.9.98.60:43787: connect: Connection refused (error 111)
10:28:06.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:06.005367 9704 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.98.60" port: 43787 } attempt: 1
10:28:06.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:06.010027 9704 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.98.60:43787: Network error: Client connection negotiation failed: client connection to 127.9.98.60:43787: connect: Connection refused (error 111)
10:28:06.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:06.091379 9704 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.98.60" port: 43787 } attempt: 2
10:28:06.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:06.095420 9704 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.98.60:43787: Network error: Client connection negotiation failed: client connection to 127.9.98.60:43787: connect: Connection refused (error 111)
10:28:06.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 10:28:06 dist-test-slave-9qlq krb5kdc[9621](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790677685, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.98.61@KRBTEST.COM for kudu/127.9.98.61@KRBTEST.COM
10:28:06.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:06.161866 9782 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.98.60" port: 43787 } has no permanent_uuid. Determining permanent_uuid...
10:28:06.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:06.164283 9732 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.98.60:43787: connect: Connection refused (error 111)
10:28:06.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:06.168031 9782 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.98.60:43787: Network error: Client connection negotiation failed: client connection to 127.9.98.60:43787: connect: Connection refused (error 111)
10:28:06.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:06.180780 9704 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.98.60" port: 43787 } attempt: 3
10:28:06.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:06.184207 9704 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.98.60:43787: Network error: Client connection negotiation failed: client connection to 127.9.98.60:43787: connect: Connection refused (error 111)
10:28:06.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:06.217612 9782 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.98.60" port: 43787 } attempt: 1
10:28:06.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:06.221164 9782 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.98.60:43787: Network error: Client connection negotiation failed: client connection to 127.9.98.60:43787: connect: Connection refused (error 111)
10:28:06.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:06.289600 9782 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.98.60" port: 43787 } attempt: 2
10:28:06.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:06.293566 9782 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.98.60:43787: Network error: Client connection negotiation failed: client connection to 127.9.98.60:43787: connect: Connection refused (error 111)
10:28:06.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:06.324651 9704 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.98.60" port: 43787 } attempt: 4
10:28:06.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:06.328684 9704 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.98.60:43787: Network error: Client connection negotiation failed: client connection to 127.9.98.60:43787: connect: Connection refused (error 111)
10:28:06.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:06.385005 9782 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.98.60" port: 43787 } attempt: 3
10:28:06.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:06.390980 9782 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.98.60:43787: Network error: Client connection negotiation failed: client connection to 127.9.98.60:43787: connect: Connection refused (error 111)
10:28:06.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:06.534451 9782 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.98.60" port: 43787 } attempt: 4
10:28:06.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:06.538168 9782 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.98.60:43787: Network error: Client connection negotiation failed: client connection to 127.9.98.60:43787: connect: Connection refused (error 111)
10:28:06.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:06.612136 9704 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.98.60" port: 43787 } attempt: 5
10:28:06.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:06.616008 9704 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.98.60:43787: Network error: Client connection negotiation failed: client connection to 127.9.98.60:43787: connect: Connection refused (error 111)
10:28:06.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:06.837507 9782 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.98.60" port: 43787 } attempt: 5
10:28:06.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:06.840622 9782 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.98.60:43787: Network error: Client connection negotiation failed: client connection to 127.9.98.60:43787: connect: Connection refused (error 111)
10:28:06.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
10:28:06.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.9.98.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/krb5kdc/kudu_127.9.98.60.keytab.
10:28:06.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.9.98.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/krb5kdc/kudu_127.9.98.60.keytab.
10:28:06.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:06.931030 9608 mini_kdc.cc:262] Time spent creating service keytab for kudu/127.9.98.60: real 1.171s user 0.005s sys 0.009s
10:28:06.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:06.933140 9608 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasks36zsq/build/tsan/bin/kudu
10:28:06.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasks36zsq/build/tsan/bin/kudu
10:28:06.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-2/wal
10:28:06.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-2/data
10:28:06.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:28:06.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:28:06.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:28:06.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:28:06.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:28:06.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:28:06.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-2/logs
10:28:06.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-2/data/info.pb
10:28:06.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:28:06.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:28:06.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:28:06.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:28:06.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:28:06.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:28:06.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:28:06.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:28:06.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
10:28:06.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:28:06.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:28:06.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:28:06.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.98.60:43787
10:28:06.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.98.60
10:28:06.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:28:06.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/krb5kdc/kudu_127.9.98.60.keytab
10:28:06.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.98.60
10:28:06.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
10:28:06.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
10:28:06.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
10:28:06.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.98.20:38125
10:28:06.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:28:06.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:28:06.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:28:06.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:28:06.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.98.62:37111,127.9.98.61:42629,127.9.98.60:43787
10:28:06.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:28:06.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37887/jwks.json
10:28:06.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
10:28:07.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:07.168421 9704 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.98.60" port: 43787 } attempt: 6
10:28:07.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:07.175163 9704 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.98.60:43787: Network error: Client connection negotiation failed: client connection to 127.9.98.60:43787: connect: Connection refused (error 111)
10:28:07.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:07.305843 9795 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
10:28:07.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:07.306362 9795 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:28:07.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:07.306958 9795 flags.cc:425] Enabled unsafe flag: --never_fsync=true
10:28:07.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:07.339520 9795 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
10:28:07.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:07.339741 9795 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
10:28:07.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:07.340189 9795 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
10:28:07.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:07.340529 9795 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
10:28:07.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:07.340822 9795 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
10:28:07.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:07.341202 9795 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
10:28:07.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:07.341449 9795 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37887/jwks.json
10:28:07.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:07.385646 9795 master_runner.cc:386] Master server non-default flags:
10:28:07.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:28:07.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.98.20:38125
10:28:07.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:28:07.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:28:07.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-2/data
10:28:07.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-2/wal
10:28:07.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
10:28:07.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:28:07.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.98.62:37111,127.9.98.61:42629,127.9.98.60:43787
10:28:07.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/krb5kdc/kudu_127.9.98.60.keytab
10:28:07.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.98.60
10:28:07.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:28:07.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:28:07.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:28:07.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.98.60:43787
10:28:07.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:28:07.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:28:07.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:28:07.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37887/jwks.json
10:28:07.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
10:28:07.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
10:28:07.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
10:28:07.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:28:07.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:28:07.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-2/data/info.pb
10:28:07.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.98.60
10:28:07.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:28:07.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:28:07.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:28:07.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:28:07.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:28:07.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:28:07.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-2/logs
10:28:07.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:28:07.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:28:07.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
10:28:07.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
10:28:07.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
10:28:07.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
10:28:07.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:28:07.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 09:27:27 UTC on ccycloud.quasar-stwief.root.comops.site
10:28:07.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679491
10:28:07.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
10:28:07.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:07.386813 9795 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:28:07.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:07.387755 9795 file_cache.cc:493] Constructed file cache file cache with capacity 419430
10:28:07.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:07.388012 9782 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.98.60" port: 43787 } attempt: 6
10:28:07.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:07.395498 9782 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.98.60:43787: Network error: Client connection negotiation failed: client connection to 127.9.98.60:43787: connect: Connection refused (error 111)
10:28:07.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:07.408675 9808 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:28:08.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:08.208639 9704 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.98.60" port: 43787 } attempt: 7
10:28:08.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:08.224375 9704 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.98.60:43787: Network error: Client connection negotiation failed: client connection to 127.9.98.60:43787: connect: Connection refused (error 111)
10:28:08.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:08.455984 9782 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.98.60" port: 43787 } attempt: 7
10:28:08.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:08.462586 9782 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.98.60:43787: Network error: Client connection negotiation failed: client connection to 127.9.98.60:43787: connect: Connection refused (error 111)
10:28:08.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:07.409134 9805 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:28:08.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:08.682014 9807 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
10:28:08.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:07.408667 9806 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:28:08.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:08.682574 9795 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
10:28:08.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:08.688701 9795 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:28:08.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:08.691741 9795 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:28:08.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:08.693149 9795 hybrid_clock.cc:648] HybridClock initialized: now 1790677688693051 us; error 136 us; skew 500 ppm
10:28:08.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:08.693988 9795 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:28:08.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 10:28:08 dist-test-slave-9qlq krb5kdc[9621](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790677688, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.98.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
10:28:08.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:08.700599 9795 init.cc:377] Logged in from keytab as kudu/127.9.98.60@KRBTEST.COM (short username kudu)
10:28:08.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:08.713024 9795 webserver.cc:460] Webserver started at http://127.9.98.60:38565/ using document root <none> and password file <none>
10:28:08.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:08.713973 9795 fs_manager.cc:359] Metadata directory not provided
10:28:08.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:08.714262 9795 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:28:08.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:08.714751 9795 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
10:28:08.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:08.719609 9795 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-2/data/instance:
10:28:08.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "115869408e2648c99dcc6a2307fe8246"
10:28:08.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 10:28:08 on dist-test-slave-9qlq"
10:28:08.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:08.720806 9795 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-2/wal/instance:
10:28:08.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "115869408e2648c99dcc6a2307fe8246"
10:28:08.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 10:28:08 on dist-test-slave-9qlq"
10:28:08.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:08.729499 9795 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.007s sys 0.001s
10:28:08.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:08.735446 9822 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
10:28:08.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:08.736760 9795 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.000s
10:28:08.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:08.737145 9795 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-2/data,/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-2/wal
10:28:08.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "115869408e2648c99dcc6a2307fe8246"
10:28:08.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 10:28:08 on dist-test-slave-9qlq"
10:28:08.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:08.737562 9795 fs_report.cc:383] FS layout report
10:28:08.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:28:08.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-2/wal
10:28:08.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-2/wal
10:28:08.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-2/data/data
10:28:08.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:28:08.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:28:08.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:28:08.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:28:08.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:28:08.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:28:08.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:28:08.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:28:08.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:28:08.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:28:08.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:28:08.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:08.783280 9795 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
10:28:08.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:08.789584 9795 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:28:08.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:08.790143 9795 kserver.cc:163] Server-wide thread pool size limit: 3276
10:28:08.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:08.866919 9795 rpc_server.cc:307] RPC server started. Bound to: 127.9.98.60:43787
10:28:08.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:08.866964 9873 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.9.98.60:43787 every 8 connection(s)
10:28:08.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:08.869906 9795 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-2/data/info.pb
10:28:08.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:08.870682 9608 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasks36zsq/build/tsan/bin/kudu as pid 9795
10:28:08.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:08.871151 9608 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-2/wal/instance
10:28:08.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:08.877919 9874 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
10:28:08.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:08.898927 9874 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.98.62" port: 37111 } has no permanent_uuid. Determining permanent_uuid...
10:28:08.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 10:28:08 dist-test-slave-9qlq krb5kdc[9621](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790677681, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.9.98.60@KRBTEST.COM
10:28:08.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 10:28:08 dist-test-slave-9qlq krb5kdc[9621](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790677688, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.98.60@KRBTEST.COM for kudu/127.9.98.62@KRBTEST.COM
10:28:08.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:08.936733 9874 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.98.61" port: 42629 } has no permanent_uuid. Determining permanent_uuid...
10:28:08.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 10:28:08 dist-test-slave-9qlq krb5kdc[9621](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790677681, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.9.98.61@KRBTEST.COM
10:28:08.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 10:28:08 dist-test-slave-9qlq krb5kdc[9621](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790677688, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.98.60@KRBTEST.COM for kudu/127.9.98.61@KRBTEST.COM
10:28:08.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:08.964896 9874 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.98.60" port: 43787 } has no permanent_uuid. Determining permanent_uuid...
10:28:08.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 10:28:08 dist-test-slave-9qlq krb5kdc[9621](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790677681, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.9.98.62@KRBTEST.COM
10:28:08.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 10:28:08 dist-test-slave-9qlq krb5kdc[9621](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790677688, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.98.60@KRBTEST.COM for kudu/127.9.98.60@KRBTEST.COM
10:28:08.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:08.993706 9608 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
10:28:09.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:08.999897 9874 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246: Bootstrap starting.
10:28:09.004 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
10:28:09.005 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
10:28:09.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:09.006688 9874 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246: Neither blocks nor log segments found. Creating new log.
10:28:09.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:09.008795 9874 log.cc:824] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246: Log is configured to *not* fsync() on all Append() calls
10:28:09.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:09.013302 9874 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246: No bootstrap required, opened a new log
10:28:09.017 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
10:28:09.019 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
10:28:09.022 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.9.98.62"
port: 37111
}
bound_http_address {
host: "127.9.98.62"
port: 41453
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.9.98.61"
port: 42629
}
bound_http_address {
host: "127.9.98.61"
port: 34199
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.9.98.60"
port: 43787
}
bound_http_address {
host: "127.9.98.60"
port: 38565
}
}
}
10:28:09.023 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
10:28:09.023 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
10:28:09.023 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
10:28:09.025 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
10:28:09.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:09.038987 9874 raft_consensus.cc:348] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d6be5e30a64a408f85d5b2b07a3cca91" member_type: VOTER last_known_addr { host: "127.9.98.62" port: 37111 } } peers { permanent_uuid: "b3a4650bfa6e4be98ec505cb9046e767" member_type: VOTER last_known_addr { host: "127.9.98.61" port: 42629 } } peers { permanent_uuid: "115869408e2648c99dcc6a2307fe8246" member_type: VOTER last_known_addr { host: "127.9.98.60" port: 43787 } }
10:28:09.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:09.039568 9874 raft_consensus.cc:374] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
10:28:09.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:09.039845 9874 raft_consensus.cc:729] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 115869408e2648c99dcc6a2307fe8246, State: Initialized, Role: FOLLOWER
10:28:09.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:09.040782 9874 consensus_queue.cc:260] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d6be5e30a64a408f85d5b2b07a3cca91" member_type: VOTER last_known_addr { host: "127.9.98.62" port: 37111 } } peers { permanent_uuid: "b3a4650bfa6e4be98ec505cb9046e767" member_type: VOTER last_known_addr { host: "127.9.98.61" port: 42629 } } peers { permanent_uuid: "115869408e2648c99dcc6a2307fe8246" member_type: VOTER last_known_addr { host: "127.9.98.60" port: 43787 } }
10:28:09.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:09.044214 9884 sys_catalog.cc:455] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d6be5e30a64a408f85d5b2b07a3cca91" member_type: VOTER last_known_addr { host: "127.9.98.62" port: 37111 } } peers { permanent_uuid: "b3a4650bfa6e4be98ec505cb9046e767" member_type: VOTER last_known_addr { host: "127.9.98.61" port: 42629 } } peers { permanent_uuid: "115869408e2648c99dcc6a2307fe8246" member_type: VOTER last_known_addr { host: "127.9.98.60" port: 43787 } } }
10:28:09.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:09.045377 9884 sys_catalog.cc:458] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246 [sys.catalog]: This master's current role is: FOLLOWER
10:28:09.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:09.047164 9874 sys_catalog.cc:564] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246 [sys.catalog]: configured and running, proceeding with master startup.
10:28:09.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 10:28:09 dist-test-slave-9qlq krb5kdc[9621](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790677689, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
10:28:09.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
10:28:09.065 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
10:28:09.068 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/krb5kdc/krb5cc
10:28:09.077 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
10:28:09.079 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/krb5kdc/krb5cc
10:28:09.080 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
10:28:09.084 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
10:28:09.086 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
10:28:09.086 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
10:28:09.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:09.096758 9896 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
10:28:09.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:09.097174 9896 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
10:28:09.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:09.077819 9795 master_runner.cc:186] Error getting master registration for 127.9.98.62:37111: OK, instance_id { permanent_uuid: "d6be5e30a64a408f85d5b2b07a3cca91" instance_seqno: 1790677683355715 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
10:28:09.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:09.113368 9795 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
10:28:09.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 10:28:09 dist-test-slave-9qlq krb5kdc[9621](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790677689, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
10:28:09.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
10:28:09.125 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
10:28:09.125 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/krb5kdc/krb5cc
10:28:09.126 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
10:28:09.127 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.9.98.61:42629
10:28:09.129 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
10:28:09.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:09.130321 9608 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasks36zsq/build/tsan/bin/kudu with pid 9710
10:28:09.152 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
10:28:09.152 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.9.98.62:37111
10:28:09.153 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
10:28:09.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:09.154271 9608 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasks36zsq/build/tsan/bin/kudu with pid 9634
10:28:09.180 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
10:28:09.181 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.9.98.60:43787
10:28:09.181 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
10:28:09.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:09.181797 9608 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasks36zsq/build/tsan/bin/kudu with pid 9795
10:28:09.202 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
10:28:09.202 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.9.98.61:42629
10:28:09.204 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
10:28:09.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:09.205911 9608 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasks36zsq/build/tsan/bin/kudu
10:28:09.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasks36zsq/build/tsan/bin/kudu
10:28:09.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-1/wal
10:28:09.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-1/data
10:28:09.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:28:09.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:28:09.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:28:09.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:28:09.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:28:09.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:28:09.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-1/logs
10:28:09.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-1/data/info.pb
10:28:09.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:28:09.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:28:09.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:28:09.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:28:09.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:28:09.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:28:09.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:28:09.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:28:09.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
10:28:09.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:28:09.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:28:09.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:28:09.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.98.61:42629
10:28:09.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.98.61
10:28:09.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34199
10:28:09.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/krb5kdc/kudu_127.9.98.61.keytab
10:28:09.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.98.61
10:28:09.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
10:28:09.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
10:28:09.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
10:28:09.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.98.20:38125
10:28:09.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:28:09.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:28:09.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:28:09.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:28:09.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.98.62:37111,127.9.98.61:42629,127.9.98.60:43787
10:28:09.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:28:09.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37887/jwks.json
10:28:09.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
10:28:09.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:09.513291 9900 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
10:28:09.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:09.513955 9900 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:28:09.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:09.514639 9900 flags.cc:425] Enabled unsafe flag: --never_fsync=true
10:28:09.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:09.544639 9900 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
10:28:09.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:09.544929 9900 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
10:28:09.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:09.545315 9900 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
10:28:09.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:09.545663 9900 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
10:28:09.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:09.546012 9900 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
10:28:09.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:09.546345 9900 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
10:28:09.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:09.546576 9900 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37887/jwks.json
10:28:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:09.584585 9900 master_runner.cc:386] Master server non-default flags:
10:28:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:28:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.98.20:38125
10:28:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:28:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:28:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-1/data
10:28:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-1/wal
10:28:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
10:28:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:28:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.98.62:37111,127.9.98.61:42629,127.9.98.60:43787
10:28:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/krb5kdc/kudu_127.9.98.61.keytab
10:28:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.98.61
10:28:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:28:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:28:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:28:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.98.61:42629
10:28:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:28:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:28:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:28:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37887/jwks.json
10:28:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
10:28:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
10:28:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
10:28:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:28:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:28:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-1/data/info.pb
10:28:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.98.61
10:28:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34199
10:28:09.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:28:09.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:28:09.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:28:09.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:28:09.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:28:09.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-1/logs
10:28:09.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:28:09.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:28:09.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
10:28:09.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
10:28:09.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
10:28:09.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
10:28:09.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:28:09.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 09:27:27 UTC on ccycloud.quasar-stwief.root.comops.site
10:28:09.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679491
10:28:09.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
10:28:09.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:09.586041 9900 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:28:09.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:09.587134 9900 file_cache.cc:493] Constructed file cache file cache with capacity 419430
10:28:09.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:09.603880 9906 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:28:10.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:09.605093 9909 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:28:10.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:09.603941 9907 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:28:10.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:10.832561 9908 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
10:28:10.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:10.832813 9900 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
10:28:10.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:10.836684 9900 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:28:10.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:10.839090 9900 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:28:10.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:10.840444 9900 hybrid_clock.cc:648] HybridClock initialized: now 1790677690840416 us; error 43 us; skew 500 ppm
10:28:10.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:10.841266 9900 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:28:10.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 10:28:10 dist-test-slave-9qlq krb5kdc[9621](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790677690, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.98.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
10:28:10.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:10.850844 9900 init.cc:377] Logged in from keytab as kudu/127.9.98.61@KRBTEST.COM (short username kudu)
10:28:10.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:10.856017 9900 webserver.cc:460] Webserver started at http://127.9.98.61:34199/ using document root <none> and password file <none>
10:28:10.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:10.857197 9900 fs_manager.cc:359] Metadata directory not provided
10:28:10.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:10.857460 9900 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:28:10.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:10.865301 9900 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.003s sys 0.001s
10:28:10.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:10.869509 9919 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
10:28:10.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:10.870460 9900 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.002s
10:28:10.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:10.870846 9900 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-1/data,/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-1/wal
10:28:10.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b3a4650bfa6e4be98ec505cb9046e767"
10:28:10.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 10:28:05 on dist-test-slave-9qlq"
10:28:10.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:10.873301 9900 fs_report.cc:383] FS layout report
10:28:10.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:28:10.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-1/wal
10:28:10.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-1/wal
10:28:10.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-1/data/data
10:28:10.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:28:10.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:28:10.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:28:10.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:28:10.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:28:10.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:28:10.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:28:10.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:28:10.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:28:10.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:28:10.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:28:10.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:10.919595 9900 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
10:28:10.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:10.926523 9900 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:28:10.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:10.927018 9900 kserver.cc:163] Server-wide thread pool size limit: 3276
10:28:11.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:11.006127 9900 rpc_server.cc:307] RPC server started. Bound to: 127.9.98.61:42629
10:28:11.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:11.006291 9970 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.9.98.61:42629 every 8 connection(s)
10:28:11.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:11.009022 9900 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-1/data/info.pb
10:28:11.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:11.015461 9608 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasks36zsq/build/tsan/bin/kudu as pid 9900
10:28:11.016 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
10:28:11.016 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.9.98.62:37111
10:28:11.016 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
10:28:11.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:11.018414 9608 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasks36zsq/build/tsan/bin/kudu
10:28:11.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasks36zsq/build/tsan/bin/kudu
10:28:11.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-0/wal
10:28:11.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-0/data
10:28:11.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:28:11.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:28:11.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:28:11.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:28:11.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:28:11.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:28:11.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-0/logs
10:28:11.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-0/data/info.pb
10:28:11.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:28:11.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:28:11.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:28:11.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:28:11.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:28:11.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:28:11.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:28:11.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:28:11.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
10:28:11.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:28:11.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:28:11.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:28:11.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.98.62:37111
10:28:11.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.98.62
10:28:11.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41453
10:28:11.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/krb5kdc/kudu_127.9.98.62.keytab
10:28:11.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.98.62
10:28:11.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
10:28:11.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
10:28:11.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
10:28:11.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.98.20:38125
10:28:11.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:28:11.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:28:11.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:28:11.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:28:11.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.98.62:37111,127.9.98.61:42629,127.9.98.60:43787
10:28:11.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:28:11.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37887/jwks.json
10:28:11.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
10:28:11.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:11.021116 9971 sys_catalog.cc:263] Verifying existing consensus state
10:28:11.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260929 10:28:11.022075 9971 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
10:28:11.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:11.022544 9900 master.cc:537] Master@127.9.98.61:42629 shutting down...
10:28:11.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:11.086709 9900 master.cc:552] Master@127.9.98.61:42629 shutdown complete.
10:28:11.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
10:28:11.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:11.330608 9973 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
10:28:11.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:11.331261 9973 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:28:11.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:11.331998 9973 flags.cc:425] Enabled unsafe flag: --never_fsync=true
10:28:11.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:11.362160 9973 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
10:28:11.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:11.362463 9973 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
10:28:11.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:11.362949 9973 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
10:28:11.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:11.363302 9973 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
10:28:11.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:11.363654 9973 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
10:28:11.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:11.363972 9973 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
10:28:11.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:11.364220 9973 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37887/jwks.json
10:28:11.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:11.402487 9973 master_runner.cc:386] Master server non-default flags:
10:28:11.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:28:11.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.98.20:38125
10:28:11.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:28:11.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:28:11.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-0/data
10:28:11.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-0/wal
10:28:11.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
10:28:11.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:28:11.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.98.62:37111,127.9.98.61:42629,127.9.98.60:43787
10:28:11.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/krb5kdc/kudu_127.9.98.62.keytab
10:28:11.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.98.62
10:28:11.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:28:11.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:28:11.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:28:11.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.98.62:37111
10:28:11.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:28:11.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:28:11.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:28:11.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37887/jwks.json
10:28:11.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
10:28:11.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
10:28:11.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
10:28:11.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:28:11.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:28:11.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-0/data/info.pb
10:28:11.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.98.62
10:28:11.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41453
10:28:11.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:28:11.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:28:11.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:28:11.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:28:11.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:28:11.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-0/logs
10:28:11.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:28:11.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:28:11.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
10:28:11.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
10:28:11.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
10:28:11.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
10:28:11.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:28:11.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 09:27:27 UTC on ccycloud.quasar-stwief.root.comops.site
10:28:11.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679491
10:28:11.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
10:28:11.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:11.403959 9973 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:28:11.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:11.405304 9973 file_cache.cc:493] Constructed file cache file cache with capacity 419430
10:28:11.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:11.424278 9979 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:28:12.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:11.424577 9980 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:28:12.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:11.424842 9982 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:28:12.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:11.427395 9973 server_base.cc:1034] running on GCE node
10:28:12.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:12.669602 9973 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:28:12.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:12.672078 9973 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:28:12.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:12.673422 9973 hybrid_clock.cc:648] HybridClock initialized: now 1790677692673410 us; error 54 us; skew 500 ppm
10:28:12.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:12.674273 9973 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:28:12.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 10:28:12 dist-test-slave-9qlq krb5kdc[9621](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790677692, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.98.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
10:28:12.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:12.679979 9973 init.cc:377] Logged in from keytab as kudu/127.9.98.62@KRBTEST.COM (short username kudu)
10:28:12.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:12.685818 9973 webserver.cc:460] Webserver started at http://127.9.98.62:41453/ using document root <none> and password file <none>
10:28:12.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:12.686920 9973 fs_manager.cc:359] Metadata directory not provided
10:28:12.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:12.687187 9973 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:28:12.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:12.695472 9973 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.006s sys 0.000s
10:28:12.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:12.701222 9990 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
10:28:12.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:12.702459 9973 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.000s sys 0.002s
10:28:12.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:12.702832 9973 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-0/data,/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-0/wal
10:28:12.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d6be5e30a64a408f85d5b2b07a3cca91"
10:28:12.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 10:28:03 on dist-test-slave-9qlq"
10:28:12.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:12.705693 9973 fs_report.cc:383] FS layout report
10:28:12.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:28:12.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-0/wal
10:28:12.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-0/wal
10:28:12.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-0/data/data
10:28:12.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:28:12.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:28:12.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:28:12.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:28:12.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:28:12.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:28:12.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:28:12.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:28:12.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:28:12.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:28:12.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:28:12.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:12.807353 9973 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
10:28:12.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:12.815305 9973 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:28:12.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:12.815907 9973 kserver.cc:163] Server-wide thread pool size limit: 3276
10:28:12.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:12.888011 9973 rpc_server.cc:307] RPC server started. Bound to: 127.9.98.62:37111
10:28:12.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:12.888139 10042 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.9.98.62:37111 every 8 connection(s)
10:28:12.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:12.890775 9973 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-0/data/info.pb
10:28:12.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:12.892180 9608 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasks36zsq/build/tsan/bin/kudu as pid 9973
10:28:12.893 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
10:28:12.893 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.9.98.60:43787
10:28:12.894 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
10:28:12.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:12.895704 9608 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasks36zsq/build/tsan/bin/kudu
10:28:12.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasks36zsq/build/tsan/bin/kudu
10:28:12.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-2/wal
10:28:12.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-2/data
10:28:12.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:28:12.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:28:12.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:28:12.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:28:12.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:28:12.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:28:12.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-2/logs
10:28:12.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-2/data/info.pb
10:28:12.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:28:12.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:28:12.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:28:12.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:28:12.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:28:12.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:28:12.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:28:12.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:28:12.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
10:28:12.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:28:12.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:28:12.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:28:12.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.98.60:43787
10:28:12.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.98.60
10:28:12.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38565
10:28:12.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/krb5kdc/kudu_127.9.98.60.keytab
10:28:12.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.98.60
10:28:12.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
10:28:12.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
10:28:12.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
10:28:12.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.98.20:38125
10:28:12.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:28:12.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:28:12.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:28:12.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:28:12.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.98.62:37111,127.9.98.61:42629,127.9.98.60:43787
10:28:12.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:28:12.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37887/jwks.json
10:28:12.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
10:28:12.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:12.902357 10043 sys_catalog.cc:263] Verifying existing consensus state
10:28:12.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260929 10:28:12.903033 10043 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
10:28:12.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:12.903434 9973 master.cc:537] Master@127.9.98.62:37111 shutting down...
10:28:12.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:12.967854 9973 master.cc:552] Master@127.9.98.62:37111 shutdown complete.
10:28:12.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
10:28:13.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:13.228482 10045 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
10:28:13.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:13.229219 10045 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:28:13.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:13.229957 10045 flags.cc:425] Enabled unsafe flag: --never_fsync=true
10:28:13.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:13.261762 10045 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
10:28:13.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:13.262045 10045 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
10:28:13.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:13.262468 10045 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
10:28:13.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:13.262809 10045 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
10:28:13.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:13.263129 10045 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
10:28:13.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:13.263447 10045 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
10:28:13.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:13.263702 10045 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37887/jwks.json
10:28:13.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:13.300065 10045 master_runner.cc:386] Master server non-default flags:
10:28:13.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:28:13.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.98.20:38125
10:28:13.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:28:13.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:28:13.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-2/data
10:28:13.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-2/wal
10:28:13.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
10:28:13.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:28:13.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.98.62:37111,127.9.98.61:42629,127.9.98.60:43787
10:28:13.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/krb5kdc/kudu_127.9.98.60.keytab
10:28:13.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.98.60
10:28:13.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:28:13.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:28:13.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:28:13.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.98.60:43787
10:28:13.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:28:13.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:28:13.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:28:13.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37887/jwks.json
10:28:13.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
10:28:13.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
10:28:13.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
10:28:13.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:28:13.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:28:13.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-2/data/info.pb
10:28:13.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.98.60
10:28:13.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38565
10:28:13.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:28:13.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:28:13.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:28:13.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:28:13.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:28:13.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-2/logs
10:28:13.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:28:13.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:28:13.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
10:28:13.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
10:28:13.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
10:28:13.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
10:28:13.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:28:13.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 09:27:27 UTC on ccycloud.quasar-stwief.root.comops.site
10:28:13.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679491
10:28:13.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
10:28:13.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:13.301581 10045 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:28:13.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:13.302883 10045 file_cache.cc:493] Constructed file cache file cache with capacity 419430
10:28:13.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:13.319370 10054 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:28:14.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:13.319373 10051 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:28:14.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:13.319540 10052 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:28:14.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:13.320910 10045 server_base.cc:1034] running on GCE node
10:28:14.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:14.545641 10045 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:28:14.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:14.548480 10045 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:28:14.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:14.549851 10045 hybrid_clock.cc:648] HybridClock initialized: now 1790677694549801 us; error 48 us; skew 500 ppm
10:28:14.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:14.550791 10045 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:28:14.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 10:28:14 dist-test-slave-9qlq krb5kdc[9621](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790677694, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.98.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
10:28:14.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:14.556594 10045 init.cc:377] Logged in from keytab as kudu/127.9.98.60@KRBTEST.COM (short username kudu)
10:28:14.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:14.561782 10045 webserver.cc:460] Webserver started at http://127.9.98.60:38565/ using document root <none> and password file <none>
10:28:14.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:14.562717 10045 fs_manager.cc:359] Metadata directory not provided
10:28:14.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:14.562975 10045 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:28:14.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:14.571029 10045 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.006s sys 0.000s
10:28:14.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:14.575367 10062 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
10:28:14.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:14.576309 10045 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
10:28:14.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:14.576691 10045 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-2/data,/tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-2/wal
10:28:14.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "115869408e2648c99dcc6a2307fe8246"
10:28:14.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 10:28:08 on dist-test-slave-9qlq"
10:28:14.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:14.578949 10045 fs_report.cc:383] FS layout report
10:28:14.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:28:14.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-2/wal
10:28:14.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-2/wal
10:28:14.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-2/data/data
10:28:14.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:28:14.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:28:14.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:28:14.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:28:14.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:28:14.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:28:14.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:28:14.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:28:14.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:28:14.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:28:14.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:28:14.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:14.614753 10045 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
10:28:14.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:14.620201 10045 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:28:14.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:14.620730 10045 kserver.cc:163] Server-wide thread pool size limit: 3276
10:28:14.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:14.699388 10045 rpc_server.cc:307] RPC server started. Bound to: 127.9.98.60:43787
10:28:14.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:14.699522 10113 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.9.98.60:43787 every 8 connection(s)
10:28:14.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:14.702399 10045 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasks36zsq/test-tmp/mini-kudu-cluster8552344738182097380/master-2/data/info.pb
10:28:14.704 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
10:28:14.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:14.703598 9608 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasks36zsq/build/tsan/bin/kudu as pid 10045
10:28:14.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:14.721956 10114 sys_catalog.cc:263] Verifying existing consensus state
10:28:14.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:14.727396 10114 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246: Bootstrap starting.
10:28:14.753 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.9.98.61:42629(127.9.98.61:42629)] connecting to peer
10:28:14.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:14.767375 10114 log.cc:824] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246: Log is configured to *not* fsync() on all Append() calls
10:28:14.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:14.771730 10114 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
10:28:14.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:14.772445 10114 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246: Bootstrap complete.
10:28:14.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:14.797813 10114 raft_consensus.cc:348] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d6be5e30a64a408f85d5b2b07a3cca91" member_type: VOTER last_known_addr { host: "127.9.98.62" port: 37111 } } peers { permanent_uuid: "b3a4650bfa6e4be98ec505cb9046e767" member_type: VOTER last_known_addr { host: "127.9.98.61" port: 42629 } } peers { permanent_uuid: "115869408e2648c99dcc6a2307fe8246" member_type: VOTER last_known_addr { host: "127.9.98.60" port: 43787 } }
10:28:14.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:14.800223 10114 raft_consensus.cc:374] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
10:28:14.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:14.800542 10114 raft_consensus.cc:729] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 115869408e2648c99dcc6a2307fe8246, State: Initialized, Role: FOLLOWER
10:28:14.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:14.801453 10114 consensus_queue.cc:260] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d6be5e30a64a408f85d5b2b07a3cca91" member_type: VOTER last_known_addr { host: "127.9.98.62" port: 37111 } } peers { permanent_uuid: "b3a4650bfa6e4be98ec505cb9046e767" member_type: VOTER last_known_addr { host: "127.9.98.61" port: 42629 } } peers { permanent_uuid: "115869408e2648c99dcc6a2307fe8246" member_type: VOTER last_known_addr { host: "127.9.98.60" port: 43787 } }
10:28:14.814 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.9.98.62:37111(127.9.98.62:37111)] connecting to peer
10:28:14.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:14.821287 10114 sys_catalog.cc:564] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246 [sys.catalog]: configured and running, proceeding with master startup.
10:28:14.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:14.819695 10119 sys_catalog.cc:455] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d6be5e30a64a408f85d5b2b07a3cca91" member_type: VOTER last_known_addr { host: "127.9.98.62" port: 37111 } } peers { permanent_uuid: "b3a4650bfa6e4be98ec505cb9046e767" member_type: VOTER last_known_addr { host: "127.9.98.61" port: 42629 } } peers { permanent_uuid: "115869408e2648c99dcc6a2307fe8246" member_type: VOTER last_known_addr { host: "127.9.98.60" port: 43787 } } }
10:28:14.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:14.825834 10119 sys_catalog.cc:458] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246 [sys.catalog]: This master's current role is: FOLLOWER
10:28:14.826 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.9.98.60:43787(127.9.98.60:43787)] connecting to peer
10:28:14.858 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:14.859 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.98.62:37111(127.9.98.62:37111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:14.860 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:14.861 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.98.61:42629(127.9.98.61:42629)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:14.862 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.98.62:37111: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:14.864 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.98.61:42629: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:14.865 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.9.98.60:43787(127.9.98.60:43787)] Successfully connected to peer
10:28:14.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:14.876380 10068 tablet.cc:2369] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246: Can't schedule compaction. Clean time has not been advanced past its initial value.
10:28:14.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:14.876644 10132 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
10:28:14.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:14.876893 10132 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
10:28:14.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:14.869295 10045 master_runner.cc:186] Error getting master registration for 127.9.98.62:37111: Network error: Client connection negotiation failed: client connection to 127.9.98.62:37111: connect: Connection refused (error 111),
10:28:14.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:14.900995 10045 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
10:28:14.950 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
10:28:14.960 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.9.98.60
10:28:15.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 10:28:15 dist-test-slave-9qlq krb5kdc[9621](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790677689, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.9.98.60@KRBTEST.COM
10:28:15.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:15.187879 10119 raft_consensus.cc:482] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
10:28:15.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:15.188452 10119 raft_consensus.cc:504] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d6be5e30a64a408f85d5b2b07a3cca91" member_type: VOTER last_known_addr { host: "127.9.98.62" port: 37111 } } peers { permanent_uuid: "b3a4650bfa6e4be98ec505cb9046e767" member_type: VOTER last_known_addr { host: "127.9.98.61" port: 42629 } } peers { permanent_uuid: "115869408e2648c99dcc6a2307fe8246" member_type: VOTER last_known_addr { host: "127.9.98.60" port: 43787 } }
10:28:15.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:15.193099 10066 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.98.62:37111: connect: Connection refused (error 111)
10:28:15.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:15.194998 10119 leader_election.cc:288] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d6be5e30a64a408f85d5b2b07a3cca91 (127.9.98.62:37111), b3a4650bfa6e4be98ec505cb9046e767 (127.9.98.61:42629)
10:28:15.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:15.206722 10066 leader_election.cc:334] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d6be5e30a64a408f85d5b2b07a3cca91 (127.9.98.62:37111): Network error: Client connection negotiation failed: client connection to 127.9.98.62:37111: connect: Connection refused (error 111)
10:28:15.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:15.208356 10066 leader_election.cc:334] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b3a4650bfa6e4be98ec505cb9046e767 (127.9.98.61:42629): Network error: Client connection negotiation failed: client connection to 127.9.98.61:42629: connect: Connection refused (error 111)
10:28:15.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:15.208711 10066 leader_election.cc:302] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 115869408e2648c99dcc6a2307fe8246; no voters: b3a4650bfa6e4be98ec505cb9046e767, d6be5e30a64a408f85d5b2b07a3cca91
10:28:15.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:15.209478 10119 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
10:28:15.215 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x9020f0b4, L:/127.0.0.1:39870 - R:/127.9.98.60:43787] using SASL/GSSAPI
10:28:15.235 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787; will retry
10:28:15.252 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.98.61:42629(127.9.98.61:42629)] connecting to peer
10:28:15.253 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.98.62:37111(127.9.98.62:37111)] connecting to peer
10:28:15.257 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:15.258 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.98.61:42629(127.9.98.61:42629)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:15.258 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.98.61:42629: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:15.265 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:15.265 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.98.62:37111(127.9.98.62:37111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:15.265 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.98.62:37111: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:15.266 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787; will retry
10:28:15.271 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.98.61:42629(127.9.98.61:42629)] connecting to peer
10:28:15.272 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.98.62:37111(127.9.98.62:37111)] connecting to peer
10:28:15.273 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:15.273 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.98.61:42629(127.9.98.61:42629)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:15.274 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.98.61:42629: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:15.276 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:15.276 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.98.62:37111(127.9.98.62:37111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:15.276 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.98.62:37111: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:15.278 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787; will retry
10:28:15.291 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.98.61:42629(127.9.98.61:42629)] connecting to peer
10:28:15.292 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.98.62:37111(127.9.98.62:37111)] connecting to peer
10:28:15.293 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:15.293 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.98.61:42629(127.9.98.61:42629)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:15.294 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.98.61:42629: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:15.299 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:15.300 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.98.62:37111(127.9.98.62:37111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:15.300 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.98.62:37111: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:15.301 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787; will retry
10:28:15.310 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.98.61:42629(127.9.98.61:42629)] connecting to peer
10:28:15.311 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.98.62:37111(127.9.98.62:37111)] connecting to peer
10:28:15.312 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:15.313 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.98.61:42629(127.9.98.61:42629)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:15.315 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.98.61:42629: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:15.317 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:15.317 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.98.62:37111(127.9.98.62:37111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:15.318 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.98.62:37111: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:15.320 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787; will retry
10:28:15.351 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.98.61:42629(127.9.98.61:42629)] connecting to peer
10:28:15.353 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.98.62:37111(127.9.98.62:37111)] connecting to peer
10:28:15.354 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:15.354 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.98.61:42629(127.9.98.61:42629)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:15.357 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.98.61:42629: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:15.358 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:15.358 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.98.62:37111(127.9.98.62:37111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:15.359 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.98.62:37111: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:15.361 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787; will retry
10:28:15.410 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.98.61:42629(127.9.98.61:42629)] connecting to peer
10:28:15.412 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.98.62:37111(127.9.98.62:37111)] connecting to peer
10:28:15.413 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:15.414 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.98.61:42629(127.9.98.61:42629)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:15.415 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.98.61:42629: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:15.420 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:15.420 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.98.62:37111(127.9.98.62:37111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:15.421 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.98.62:37111: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:15.421 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787; will retry
10:28:15.451 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.98.61:42629(127.9.98.61:42629)] connecting to peer
10:28:15.453 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.98.62:37111(127.9.98.62:37111)] connecting to peer
10:28:15.455 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:15.456 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.98.61:42629(127.9.98.61:42629)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:15.456 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.98.61:42629: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:15.457 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:15.457 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.98.62:37111(127.9.98.62:37111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:15.458 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.98.62:37111: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:15.460 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787; will retry
10:28:15.491 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.98.61:42629(127.9.98.61:42629)] connecting to peer
10:28:15.492 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.98.62:37111(127.9.98.62:37111)] connecting to peer
10:28:15.494 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:15.494 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.98.61:42629(127.9.98.61:42629)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:15.495 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.98.61:42629: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:15.496 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:15.496 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.98.62:37111(127.9.98.62:37111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:15.497 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.98.62:37111: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:15.500 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787; will retry
10:28:15.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:15.881841 10132 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
10:28:15.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:15.882187 10132 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
10:28:16.011 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.98.61:42629(127.9.98.61:42629)] connecting to peer
10:28:16.013 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:16.013 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.98.61:42629(127.9.98.61:42629)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:16.014 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.98.62:37111(127.9.98.62:37111)] connecting to peer
10:28:16.014 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.98.61:42629: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:16.016 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:16.017 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.98.62:37111(127.9.98.62:37111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:16.017 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.98.62:37111: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:16.019 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787; will retry
10:28:16.030 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.98.61:42629(127.9.98.61:42629)] connecting to peer
10:28:16.032 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.98.62:37111(127.9.98.62:37111)] connecting to peer
10:28:16.032 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:16.033 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.98.61:42629(127.9.98.61:42629)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:16.033 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.98.61:42629: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:16.034 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:16.034 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.98.62:37111(127.9.98.62:37111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:16.034 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.98.62:37111: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:16.037 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787; will retry
10:28:16.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:16.885124 10132 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
10:28:16.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:16.885371 10132 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
10:28:17.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:17.286614 10139 raft_consensus.cc:482] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
10:28:17.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:17.286967 10139 raft_consensus.cc:504] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d6be5e30a64a408f85d5b2b07a3cca91" member_type: VOTER last_known_addr { host: "127.9.98.62" port: 37111 } } peers { permanent_uuid: "b3a4650bfa6e4be98ec505cb9046e767" member_type: VOTER last_known_addr { host: "127.9.98.61" port: 42629 } } peers { permanent_uuid: "115869408e2648c99dcc6a2307fe8246" member_type: VOTER last_known_addr { host: "127.9.98.60" port: 43787 } }
10:28:17.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:17.288662 10139 leader_election.cc:288] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d6be5e30a64a408f85d5b2b07a3cca91 (127.9.98.62:37111), b3a4650bfa6e4be98ec505cb9046e767 (127.9.98.61:42629)
10:28:17.291 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.98.61:42629(127.9.98.61:42629)] connecting to peer
10:28:17.292 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.98.62:37111(127.9.98.62:37111)] connecting to peer
10:28:17.294 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:17.294 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.98.61:42629(127.9.98.61:42629)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:17.295 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.98.61:42629: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:17.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:17.297807 10066 leader_election.cc:334] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d6be5e30a64a408f85d5b2b07a3cca91 (127.9.98.62:37111): Network error: Client connection negotiation failed: client connection to 127.9.98.62:37111: connect: Connection refused (error 111)
10:28:17.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:17.298481 10066 leader_election.cc:334] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b3a4650bfa6e4be98ec505cb9046e767 (127.9.98.61:42629): Network error: Client connection negotiation failed: client connection to 127.9.98.61:42629: connect: Connection refused (error 111)
10:28:17.299 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:17.299 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.98.62:37111(127.9.98.62:37111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:17.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:17.299104 10066 leader_election.cc:302] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 115869408e2648c99dcc6a2307fe8246; no voters: b3a4650bfa6e4be98ec505cb9046e767, d6be5e30a64a408f85d5b2b07a3cca91
10:28:17.299 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.98.62:37111: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:17.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:17.299877 10139 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
10:28:17.301 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787; will retry
10:28:17.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:17.888037 10132 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
10:28:17.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:17.888288 10132 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
10:28:18.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:18.891139 10132 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
10:28:18.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:18.891391 10132 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
10:28:19.131 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.98.61:42629(127.9.98.61:42629)] connecting to peer
10:28:19.132 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.98.62:37111(127.9.98.62:37111)] connecting to peer
10:28:19.133 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:19.134 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.98.61:42629(127.9.98.61:42629)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:19.134 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.98.61:42629: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:19.134 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:19.135 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.98.62:37111(127.9.98.62:37111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:19.135 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.98.62:37111: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:19.137 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787; will retry
10:28:19.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:19.894080 10132 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
10:28:19.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:19.894328 10132 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
10:28:19.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:19.903652 10045 master_runner.cc:186] Error getting master registration for 127.9.98.62:37111: Network error: Client connection negotiation failed: client connection to 127.9.98.62:37111: connect: Connection refused (error 111),
10:28:19.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:19.904141 10045 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
10:28:20.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:20.321672 10148 raft_consensus.cc:482] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
10:28:20.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:20.322072 10148 raft_consensus.cc:504] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d6be5e30a64a408f85d5b2b07a3cca91" member_type: VOTER last_known_addr { host: "127.9.98.62" port: 37111 } } peers { permanent_uuid: "b3a4650bfa6e4be98ec505cb9046e767" member_type: VOTER last_known_addr { host: "127.9.98.61" port: 42629 } } peers { permanent_uuid: "115869408e2648c99dcc6a2307fe8246" member_type: VOTER last_known_addr { host: "127.9.98.60" port: 43787 } }
10:28:20.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:20.323706 10148 leader_election.cc:288] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d6be5e30a64a408f85d5b2b07a3cca91 (127.9.98.62:37111), b3a4650bfa6e4be98ec505cb9046e767 (127.9.98.61:42629)
10:28:20.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:20.325803 10066 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.98.62:37111: connect: Connection refused (error 111) [suppressed 3 similar messages]
10:28:20.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:20.329274 10066 leader_election.cc:334] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d6be5e30a64a408f85d5b2b07a3cca91 (127.9.98.62:37111): Network error: Client connection negotiation failed: client connection to 127.9.98.62:37111: connect: Connection refused (error 111)
10:28:20.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:20.329746 10066 leader_election.cc:334] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b3a4650bfa6e4be98ec505cb9046e767 (127.9.98.61:42629): Network error: Client connection negotiation failed: client connection to 127.9.98.61:42629: connect: Connection refused (error 111)
10:28:20.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:20.330015 10066 leader_election.cc:302] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 115869408e2648c99dcc6a2307fe8246; no voters: b3a4650bfa6e4be98ec505cb9046e767, d6be5e30a64a408f85d5b2b07a3cca91
10:28:20.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:20.330570 10148 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
10:28:20.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:20.897096 10132 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
10:28:20.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:20.897388 10132 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
10:28:21.051 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.98.61:42629(127.9.98.61:42629)] connecting to peer
10:28:21.053 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.98.62:37111(127.9.98.62:37111)] connecting to peer
10:28:21.054 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:21.054 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.98.61:42629(127.9.98.61:42629)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:21.054 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.98.61:42629: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:21.055 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:21.055 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.98.62:37111(127.9.98.62:37111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:21.055 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.98.62:37111: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:21.057 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787; will retry
10:28:21.111 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.98.61:42629(127.9.98.61:42629)] connecting to peer
10:28:21.112 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.98.62:37111(127.9.98.62:37111)] connecting to peer
10:28:21.114 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:21.114 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.98.61:42629(127.9.98.61:42629)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:21.114 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.98.61:42629: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:21.115 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:21.115 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.98.62:37111(127.9.98.62:37111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:21.115 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.98.62:37111: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:21.117 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787; will retry
10:28:21.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:21.900313 10132 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
10:28:21.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:21.900583 10132 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
10:28:21.951 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.98.61:42629(127.9.98.61:42629)] connecting to peer
10:28:21.952 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.98.62:37111(127.9.98.62:37111)] connecting to peer
10:28:21.953 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:21.953 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.98.61:42629(127.9.98.61:42629)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:21.953 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.98.61:42629: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:21.956 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:21.956 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.98.62:37111(127.9.98.62:37111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:21.956 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.98.62:37111: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:21.958 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787; will retry
10:28:22.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:22.627575 10151 raft_consensus.cc:482] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
10:28:22.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:22.627944 10151 raft_consensus.cc:504] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d6be5e30a64a408f85d5b2b07a3cca91" member_type: VOTER last_known_addr { host: "127.9.98.62" port: 37111 } } peers { permanent_uuid: "b3a4650bfa6e4be98ec505cb9046e767" member_type: VOTER last_known_addr { host: "127.9.98.61" port: 42629 } } peers { permanent_uuid: "115869408e2648c99dcc6a2307fe8246" member_type: VOTER last_known_addr { host: "127.9.98.60" port: 43787 } }
10:28:22.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:22.629311 10151 leader_election.cc:288] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d6be5e30a64a408f85d5b2b07a3cca91 (127.9.98.62:37111), b3a4650bfa6e4be98ec505cb9046e767 (127.9.98.61:42629)
10:28:22.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:22.634253 10066 leader_election.cc:334] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d6be5e30a64a408f85d5b2b07a3cca91 (127.9.98.62:37111): Network error: Client connection negotiation failed: client connection to 127.9.98.62:37111: connect: Connection refused (error 111)
10:28:22.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:22.634828 10066 leader_election.cc:334] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b3a4650bfa6e4be98ec505cb9046e767 (127.9.98.61:42629): Network error: Client connection negotiation failed: client connection to 127.9.98.61:42629: connect: Connection refused (error 111)
10:28:22.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:22.635124 10066 leader_election.cc:302] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 115869408e2648c99dcc6a2307fe8246; no voters: b3a4650bfa6e4be98ec505cb9046e767, d6be5e30a64a408f85d5b2b07a3cca91
10:28:22.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:22.635685 10151 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
10:28:22.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:22.903589 10132 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
10:28:22.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:22.903877 10132 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
10:28:23.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:23.907380 10132 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
10:28:23.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:23.907757 10132 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
10:28:24.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 10:28:24 dist-test-slave-9qlq krb5kdc[9621](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790677704, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.98.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
10:28:24.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:24.560231 10058 init.cc:295] Successfully reacquired a new kerberos TGT
10:28:24.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:24.907366 10045 master_runner.cc:186] Error getting master registration for 127.9.98.62:37111: Network error: Client connection negotiation failed: client connection to 127.9.98.62:37111: connect: Connection refused (error 111),
10:28:24.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:24.907829 10045 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
10:28:24.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:24.910473 10132 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
10:28:24.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:24.910660 10132 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
10:28:25.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:25.737217 10158 raft_consensus.cc:482] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
10:28:25.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:25.737684 10158 raft_consensus.cc:504] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d6be5e30a64a408f85d5b2b07a3cca91" member_type: VOTER last_known_addr { host: "127.9.98.62" port: 37111 } } peers { permanent_uuid: "b3a4650bfa6e4be98ec505cb9046e767" member_type: VOTER last_known_addr { host: "127.9.98.61" port: 42629 } } peers { permanent_uuid: "115869408e2648c99dcc6a2307fe8246" member_type: VOTER last_known_addr { host: "127.9.98.60" port: 43787 } }
10:28:25.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:25.739578 10158 leader_election.cc:288] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d6be5e30a64a408f85d5b2b07a3cca91 (127.9.98.62:37111), b3a4650bfa6e4be98ec505cb9046e767 (127.9.98.61:42629)
10:28:25.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:25.742602 10066 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.98.62:37111: connect: Connection refused (error 111) [suppressed 3 similar messages]
10:28:25.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:25.745656 10066 leader_election.cc:334] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d6be5e30a64a408f85d5b2b07a3cca91 (127.9.98.62:37111): Network error: Client connection negotiation failed: client connection to 127.9.98.62:37111: connect: Connection refused (error 111)
10:28:25.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:25.746326 10066 leader_election.cc:334] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b3a4650bfa6e4be98ec505cb9046e767 (127.9.98.61:42629): Network error: Client connection negotiation failed: client connection to 127.9.98.61:42629: connect: Connection refused (error 111)
10:28:25.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:25.746652 10066 leader_election.cc:302] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 115869408e2648c99dcc6a2307fe8246; no voters: b3a4650bfa6e4be98ec505cb9046e767, d6be5e30a64a408f85d5b2b07a3cca91
10:28:25.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:25.747258 10158 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
10:28:25.891 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.98.61:42629(127.9.98.61:42629)] connecting to peer
10:28:25.893 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.98.62:37111(127.9.98.62:37111)] connecting to peer
10:28:25.894 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:25.894 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.98.61:42629(127.9.98.61:42629)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:25.894 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.98.61:42629: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:25.895 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:25.895 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.98.62:37111(127.9.98.62:37111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:25.895 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.98.62:37111: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:25.898 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787; will retry
10:28:25.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:25.913360 10132 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
10:28:25.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:25.913597 10132 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
10:28:26.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:26.916210 10132 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
10:28:26.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:26.916460 10132 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
10:28:27.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:27.919340 10132 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
10:28:27.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:27.919608 10132 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
10:28:28.031 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.98.61:42629(127.9.98.61:42629)] connecting to peer
10:28:28.033 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.98.62:37111(127.9.98.62:37111)] connecting to peer
10:28:28.034 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:28.035 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.98.61:42629(127.9.98.61:42629)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:28.035 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.98.61:42629: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:28.035 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:28.036 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.98.62:37111(127.9.98.62:37111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:28.036 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.98.62:37111: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:28.038 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787; will retry
10:28:28.511 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.98.61:42629(127.9.98.61:42629)] connecting to peer
10:28:28.513 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.98.62:37111(127.9.98.62:37111)] connecting to peer
10:28:28.514 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:28.514 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.98.61:42629(127.9.98.61:42629)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:28.514 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.98.61:42629: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:28.516 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:28.516 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.98.62:37111(127.9.98.62:37111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:28.516 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.98.62:37111: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:28.518 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787; will retry
10:28:28.791 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.98.61:42629(127.9.98.61:42629)] connecting to peer
10:28:28.793 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.98.62:37111(127.9.98.62:37111)] connecting to peer
10:28:28.793 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:28.794 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.98.61:42629(127.9.98.61:42629)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:28.794 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.98.61:42629: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:28.795 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:28.795 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.98.62:37111(127.9.98.62:37111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:28.795 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.98.62:37111: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:28.798 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787; will retry
10:28:28.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:28.922413 10132 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
10:28:28.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:28.922648 10132 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
10:28:29.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:29.910293 10045 master_runner.cc:186] Error getting master registration for 127.9.98.62:37111: Network error: Client connection negotiation failed: client connection to 127.9.98.62:37111: connect: Connection refused (error 111),
10:28:29.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:29.910655 10045 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
10:28:29.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:29.925806 10132 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
10:28:29.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:29.926026 10132 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
10:28:30.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:30.928722 10132 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
10:28:30.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:30.928969 10132 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
10:28:31.551 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.98.61:42629(127.9.98.61:42629)] connecting to peer
10:28:31.552 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.98.62:37111(127.9.98.62:37111)] connecting to peer
10:28:31.553 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:31.553 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.98.61:42629(127.9.98.61:42629)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:31.553 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.98.61:42629: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:31.554 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:31.554 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.98.62:37111(127.9.98.62:37111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:31.554 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.98.62:37111: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:31.558 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787; will retry
10:28:31.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:31.931587 10132 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
10:28:31.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:31.931825 10132 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
10:28:32.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:32.618348 10171 raft_consensus.cc:482] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
10:28:32.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:32.618673 10171 raft_consensus.cc:504] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d6be5e30a64a408f85d5b2b07a3cca91" member_type: VOTER last_known_addr { host: "127.9.98.62" port: 37111 } } peers { permanent_uuid: "b3a4650bfa6e4be98ec505cb9046e767" member_type: VOTER last_known_addr { host: "127.9.98.61" port: 42629 } } peers { permanent_uuid: "115869408e2648c99dcc6a2307fe8246" member_type: VOTER last_known_addr { host: "127.9.98.60" port: 43787 } }
10:28:32.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:32.620112 10171 leader_election.cc:288] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d6be5e30a64a408f85d5b2b07a3cca91 (127.9.98.62:37111), b3a4650bfa6e4be98ec505cb9046e767 (127.9.98.61:42629)
10:28:32.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:32.622387 10066 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.98.62:37111: connect: Connection refused (error 111) [suppressed 1 similar messages]
10:28:32.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:32.625152 10066 leader_election.cc:334] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d6be5e30a64a408f85d5b2b07a3cca91 (127.9.98.62:37111): Network error: Client connection negotiation failed: client connection to 127.9.98.62:37111: connect: Connection refused (error 111)
10:28:32.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:32.625671 10066 leader_election.cc:334] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b3a4650bfa6e4be98ec505cb9046e767 (127.9.98.61:42629): Network error: Client connection negotiation failed: client connection to 127.9.98.61:42629: connect: Connection refused (error 111)
10:28:32.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:32.625979 10066 leader_election.cc:302] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 115869408e2648c99dcc6a2307fe8246; no voters: b3a4650bfa6e4be98ec505cb9046e767, d6be5e30a64a408f85d5b2b07a3cca91
10:28:32.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:32.626531 10171 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
10:28:32.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:32.934897 10132 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
10:28:32.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:32.935223 10132 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
10:28:33.751 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.98.61:42629(127.9.98.61:42629)] connecting to peer
10:28:33.752 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.98.62:37111(127.9.98.62:37111)] connecting to peer
10:28:33.752 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:33.752 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.98.61:42629(127.9.98.61:42629)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:33.753 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.98.61:42629: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:33.754 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:33.754 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.98.62:37111(127.9.98.62:37111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:33.754 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.98.62:37111: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:33.756 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787; will retry
10:28:33.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:33.938596 10132 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
10:28:33.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:33.938952 10132 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
10:28:34.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 10:28:34 dist-test-slave-9qlq krb5kdc[9621](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790677714, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.98.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
10:28:34.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:34.562659 10058 init.cc:295] Successfully reacquired a new kerberos TGT
10:28:34.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:34.913918 10045 master_runner.cc:186] Error getting master registration for 127.9.98.62:37111: Network error: Client connection negotiation failed: client connection to 127.9.98.62:37111: connect: Connection refused (error 111),
10:28:34.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:34.914322 10045 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
10:28:34.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:34.941591 10132 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
10:28:34.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:34.941840 10132 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
10:28:35.051 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.98.61:42629(127.9.98.61:42629)] connecting to peer
10:28:35.052 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.98.62:37111(127.9.98.62:37111)] connecting to peer
10:28:35.053 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:35.053 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.98.61:42629(127.9.98.61:42629)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:35.053 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.98.61:42629: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:35.054 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:35.054 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.98.62:37111(127.9.98.62:37111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:35.054 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.98.62:37111: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:35.056 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787; will retry
10:28:35.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:35.944696 10132 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
10:28:35.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:35.944936 10132 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
10:28:36.411 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.98.61:42629(127.9.98.61:42629)] connecting to peer
10:28:36.412 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.98.62:37111(127.9.98.62:37111)] connecting to peer
10:28:36.412 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:36.413 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.98.61:42629(127.9.98.61:42629)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:36.413 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.98.61:42629: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:36.414 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:36.414 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.98.62:37111(127.9.98.62:37111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:36.414 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.98.62:37111: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:36.417 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787; will retry
10:28:36.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:36.947889 10132 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
10:28:36.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:36.948127 10132 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
10:28:37.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:37.253885 10182 raft_consensus.cc:482] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
10:28:37.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:37.254309 10182 raft_consensus.cc:504] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d6be5e30a64a408f85d5b2b07a3cca91" member_type: VOTER last_known_addr { host: "127.9.98.62" port: 37111 } } peers { permanent_uuid: "b3a4650bfa6e4be98ec505cb9046e767" member_type: VOTER last_known_addr { host: "127.9.98.61" port: 42629 } } peers { permanent_uuid: "115869408e2648c99dcc6a2307fe8246" member_type: VOTER last_known_addr { host: "127.9.98.60" port: 43787 } }
10:28:37.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:37.255846 10182 leader_election.cc:288] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d6be5e30a64a408f85d5b2b07a3cca91 (127.9.98.62:37111), b3a4650bfa6e4be98ec505cb9046e767 (127.9.98.61:42629)
10:28:37.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:37.262506 10066 leader_election.cc:334] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d6be5e30a64a408f85d5b2b07a3cca91 (127.9.98.62:37111): Network error: Client connection negotiation failed: client connection to 127.9.98.62:37111: connect: Connection refused (error 111)
10:28:37.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:37.263046 10066 leader_election.cc:334] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b3a4650bfa6e4be98ec505cb9046e767 (127.9.98.61:42629): Network error: Client connection negotiation failed: client connection to 127.9.98.61:42629: connect: Connection refused (error 111)
10:28:37.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:37.263365 10066 leader_election.cc:302] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 115869408e2648c99dcc6a2307fe8246; no voters: b3a4650bfa6e4be98ec505cb9046e767, d6be5e30a64a408f85d5b2b07a3cca91
10:28:37.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:37.264040 10182 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
10:28:37.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:37.950968 10132 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
10:28:37.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:37.951228 10132 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
10:28:38.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:38.954056 10132 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
10:28:38.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:38.954320 10132 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
10:28:39.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:39.916730 10045 master_runner.cc:186] Error getting master registration for 127.9.98.62:37111: Network error: Client connection negotiation failed: client connection to 127.9.98.62:37111: connect: Connection refused (error 111),
10:28:39.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:39.917052 10045 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
10:28:39.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:39.957041 10132 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
10:28:39.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:39.957299 10132 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
10:28:40.311 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.98.61:42629(127.9.98.61:42629)] connecting to peer
10:28:40.312 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.98.62:37111(127.9.98.62:37111)] connecting to peer
10:28:40.313 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:40.313 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.98.61:42629(127.9.98.61:42629)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:40.313 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.98.61:42629: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:40.314 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:40.314 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.98.62:37111(127.9.98.62:37111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:40.314 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.98.62:37111: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:40.317 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787; will retry
10:28:40.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:40.960489 10132 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
10:28:40.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:40.960747 10132 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
10:28:41.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:41.963634 10132 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
10:28:41.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:41.963891 10132 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
10:28:41.971 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.98.61:42629(127.9.98.61:42629)] connecting to peer
10:28:41.972 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.98.62:37111(127.9.98.62:37111)] connecting to peer
10:28:41.972 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:41.972 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.98.61:42629(127.9.98.61:42629)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:41.973 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.98.61:42629: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:41.973 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:41.973 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.98.62:37111(127.9.98.62:37111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:41.974 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.98.62:37111: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:41.976 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787; will retry
10:28:42.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:42.967294 10132 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
10:28:42.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:42.967639 10132 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
10:28:43.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:43.971138 10132 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 115869408e2648c99dcc6a2307fe8246: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
10:28:43.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 10:28:43.971448 10132 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
10:28:44.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 10:28:44 dist-test-slave-9qlq krb5kdc[9621](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790677724, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.98.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
10:28:44.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:44.565143 10058 init.cc:295] Successfully reacquired a new kerberos TGT
10:28:44.591 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.98.61:42629(127.9.98.61:42629)] connecting to peer
10:28:44.592 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.98.62:37111(127.9.98.62:37111)] connecting to peer
10:28:44.592 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:44.592 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.98.61:42629(127.9.98.61:42629)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:44.593 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.98.61:42629: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629
10:28:44.594 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:44.594 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.98.62:37111(127.9.98.62:37111)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:44.594 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.98.62:37111: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
10:28:44.597 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787; will retry
10:28:44.601 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=29888), Traces: [0ms] refreshing cache from master, [30ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.61:42629, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.62:37111, [111ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.60:43787, [146ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.61:42629: Network error: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629, [145ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.62:37111: Network error: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [516ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.60:43787: OK, [520ms] delaying RPC due to: Service unavailable: Master config (127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629, [535ms] refreshing cache from master, [536ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.61:42629, [538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.62:37111, [539ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.60:43787, [543ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.61:42629: Network error: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629, [548ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.60:43787: OK, [550ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.62:37111: Network error: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [551ms] delaying RPC due to: Service unavailable: Master config (127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [555ms] refreshing cache from master, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.61:42629, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.62:37111, [558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.60:43787, [558ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.61:42629: Network error: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629, [561ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.62:37111: Network error: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [562ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.60:43787: OK, [564ms] delaying RPC due to: Service unavailable: Master config (127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [575ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.61:42629, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.62:37111, [578ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.61:42629: Network error: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629, [579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.60:43787, [584ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.60:43787: OK, [585ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.62:37111: Network error: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [586ms] delaying RPC due to: Service unavailable: Master config (127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [595ms] refreshing cache from master, [595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.61:42629, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.62:37111, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.60:43787, [599ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.61:42629: Network error: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629, [602ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.62:37111: Network error: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [605ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.60:43787: OK, [606ms] delaying RPC due to: Service unavailable: Master config (127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [636ms] refreshing cache from master, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.61:42629, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.62:37111, [639ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.60:43787, [639ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.61:42629: Network error: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629, [643ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.62:37111: Network error: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [645ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.60:43787: OK, [649ms] delaying RPC due to: Service unavailable: Master config (127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [695ms] refreshing cache from master, [695ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.61:42629, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.62:37111, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.60:43787, [699ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.61:42629: Network error: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629, [704ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.60:43787: OK, [705ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.62:37111: Network error: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [706ms] delaying RPC due to: Service unavailable: Master config (127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [736ms] refreshing cache from master, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.61:42629, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.62:37111, [739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.60:43787, [741ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.61:42629: Network error: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629, [742ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.62:37111: Network error: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [744ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.60:43787: OK, [745ms] delaying RPC due to: Service unavailable: Master config (127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [776ms] refreshing cache from master, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.61:42629, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.62:37111, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.60:43787, [779ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.61:42629: Network error: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629, [782ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.62:37111: Network error: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [784ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.60:43787: OK, [785ms] delaying RPC due to: Service unavailable: Master config (127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [1295ms] refreshing cache from master, [1296ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.61:42629, [1297ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.62:37111, [1298ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.61:42629: Network error: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629, [1300ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.60:43787, [1302ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.62:37111: Network error: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [1303ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.60:43787: OK, [1304ms] delaying RPC due to: Service unavailable: Master config (127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [1315ms] refreshing cache from master, [1315ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.61:42629, [1316ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.62:37111, [1317ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.60:43787, [1318ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.61:42629: Network error: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629, [1319ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.62:37111: Network error: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [1322ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.60:43787: OK, [1322ms] delaying RPC due to: Service unavailable: Master config (127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [2576ms] refreshing cache from master, [2576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.61:42629, [2577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.62:37111, [2578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.60:43787, [2580ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.61:42629: Network error: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629, [2584ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.62:37111: Network error: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [2585ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.60:43787: OK, [2586ms] delaying RPC due to: Service unavailable: Master config (127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [4416ms] refreshing cache from master, [4416ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.61:42629, [4417ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.62:37111, [4418ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.60:43787, [4419ms] too many traces: truncated at 100 traces, deferred=Deferred@889477129(state=PENDING, result=null, callback=(continuation of Deferred@1891718423 after retry RPC after error@1551332473) -> (continuation of Deferred@1319563932 after retry RPC after error@1509244753) -> (continuation of Deferred@593932437 after retry RPC after error@1154862140) -> (continuation of Deferred@1025438760 after retry RPC after error@1306610541) -> (continuation of Deferred@1536534620 after retry RPC after error@827013950) -> (continuation of Deferred@2067674763 after retry RPC after error@1011069090) -> (continuation of Deferred@1247946814 after retry RPC after error@580413575) -> (continuation of Deferred@672700465 after retry RPC after error@167766832) -> (continuation of Deferred@1376190953 after retry RPC after error@867862398) -> (continuation of Deferred@1594837032 after retry RPC after error@702516882) -> (continuation of Deferred@1136511497 after retry RPC after error@618345591) -> (continuation of Deferred@28337868 after retry RPC after error@1422836598) -> (continuation of Deferred@2039391844 after retry RPC after error@76238254) -> (continuation of Deferred@101703157 after retry RPC after error@620260145) -> (continuation of Deferred@684278860 after retry RPC after error@1995545286) -> (continuation of Deferred@250155416 after retry RPC after error@1764709015) -> (continuation of Deferred@195465434 after retry RPC after error@1912282306) -> (continuation of Deferred@936945332 after retry RPC after error@611381554) -> (continuation of Deferred@460297510 after retry RPC after error@853247877) -> (continuation of Deferred@1086618220 after retry RPC after error@1671359015) -> (continuation of Deferred@71275991 after retry RPC after error@1790365050) -> (continuation of Deferred@163765594 after retry RPC after error@488231319) -> (continuation of Deferred@1710259918 after retry RPC after error@963748375) -> (continuation of Deferred@650547310 after retry RPC after error@1962960968) -> (continuation of Deferred@325086760 after retry RPC after error@657790793) -> (continuation of Deferred@895661601 after retry RPC after error@1129765613), errback=(continuation of Deferred@1891718423 after retry RPC after error@1551332473) -> (continuation of Deferred@1319563932 after retry RPC after error@1509244753) -> (continuation of Deferred@593932437 after retry RPC after error@1154862140) -> (continuation of Deferred@1025438760 after retry RPC after error@1306610541) -> (continuation of Deferred@1536534620 after retry RPC after error@827013950) -> (continuation of Deferred@2067674763 after retry RPC after error@1011069090) -> (continuation of Deferred@1247946814 after retry RPC after error@580413575) -> (continuation of Deferred@672700465 after retry RPC after error@167766832) -> (continuation of Deferred@1376190953 after retry RPC after error@867862398) -> (continuation of Deferred@1594837032 after retry RPC after error@702516882) -> (continuation of Deferred@1136511497 after retry RPC after error@618345591) -> (continuation of Deferred@28337868 after retry RPC after error@1422836598) -> (continuation of Deferred@2039391844 after retry RPC after error@76238254) -> (continuation of Deferred@101703157 after retry RPC after error@620260145) -> (continuation of Deferred@684278860 after retry RPC after error@1995545286) -> (continuation of Deferred@250155416 after retry RPC after error@1764709015) -> (continuation of Deferred@195465434 after retry RPC after error@1912282306) -> (continuation of Deferred@936945332 after retry RPC after error@611381554) -> (continuation of Deferred@460297510 after retry RPC after error@853247877) -> (continuation of Deferred@1086618220 after retry RPC after error@1671359015) -> (continuation of Deferred@71275991 after retry RPC after error@1790365050) -> (continuation of Deferred@163765594 after retry RPC after error@488231319) -> (continuation of Deferred@1710259918 after retry RPC after error@963748375) -> (continuation of Deferred@650547310 after retry RPC after error@1962960968) -> (continuation of Deferred@325086760 after retry RPC after error@657790793) -> (continuation of Deferred@895661601 after retry RPC after error@1129765613)))
10:28:44.606 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.9.98.61:42629(127.9.98.61:42629)] disconnecting while in state TERMINATED
10:28:44.607 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.9.98.62:37111(127.9.98.62:37111)] disconnecting while in state TERMINATED
10:28:44.607 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.9.98.60:43787(127.9.98.60:43787)] disconnecting while in state READY
10:28:44.611 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
10:28:44.613 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.9.98.60:43787(127.9.98.60:43787)] handling channelInactive
10:28:44.613 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.98.60:43787(127.9.98.60:43787)] cleaning up while in state READY due to: connection closed
10:28:44.615 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
10:28:44.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:44.618096 9608 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasks36zsq/build/tsan/bin/kudu with pid 10045
10:28:45.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T10:28:45Z chronyd exiting
10:28:46.809 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
10:28:46.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:46.810561 8886 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasks36zsq/build/tsan/bin/kudu with pid 9182
10:28:46.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:46.833045 8886 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasks36zsq/build/tsan/bin/kudu with pid 9322
10:28:46.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:46.857913 8886 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasks36zsq/build/tsan/bin/kudu with pid 9467
10:28:46.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:46.884640 8886 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasks36zsq/build/tsan/bin/kudu with pid 8898
10:28:46.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:46.914485 8886 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasks36zsq/build/tsan/bin/kudu with pid 8968
10:28:46.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 10:28:46.942567 8886 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasks36zsq/build/tsan/bin/kudu with pid 9043
10:28:48.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T10:28:48Z chronyd exiting
10:28:49.106 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=29888), Traces: [0ms] refreshing cache from master, [30ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.61:42629, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.62:37111, [111ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.60:43787, [146ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.61:42629: Network error: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629, [145ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.62:37111: Network error: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [516ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.60:43787: OK, [520ms] delaying RPC due to: Service unavailable: Master config (127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629, [535ms] refreshing cache from master, [536ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.61:42629, [538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.62:37111, [539ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.60:43787, [543ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.61:42629: Network error: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629, [548ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.60:43787: OK, [550ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.62:37111: Network error: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [551ms] delaying RPC due to: Service unavailable: Master config (127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [555ms] refreshing cache from master, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.61:42629, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.62:37111, [558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.60:43787, [558ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.61:42629: Network error: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629, [561ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.62:37111: Network error: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [562ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.60:43787: OK, [564ms] delaying RPC due to: Service unavailable: Master config (127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [575ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.61:42629, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.62:37111, [578ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.61:42629: Network error: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629, [579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.60:43787, [584ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.60:43787: OK, [585ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.62:37111: Network error: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [586ms] delaying RPC due to: Service unavailable: Master config (127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [595ms] refreshing cache from master, [595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.61:42629, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.62:37111, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.60:43787, [599ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.61:42629: Network error: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629, [602ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.62:37111: Network error: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [605ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.60:43787: OK, [606ms] delaying RPC due to: Service unavailable: Master config (127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [636ms] refreshing cache from master, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.61:42629, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.62:37111, [639ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.60:43787, [639ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.61:42629: Network error: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629, [643ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.62:37111: Network error: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [645ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.60:43787: OK, [649ms] delaying RPC due to: Service unavailable: Master config (127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [695ms] refreshing cache from master, [695ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.61:42629, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.62:37111, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.60:43787, [699ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.61:42629: Network error: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629, [704ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.60:43787: OK, [705ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.62:37111: Network error: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [706ms] delaying RPC due to: Service unavailable: Master config (127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [736ms] refreshing cache from master, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.61:42629, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.62:37111, [739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.60:43787, [741ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.61:42629: Network error: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629, [742ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.62:37111: Network error: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [744ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.60:43787: OK, [745ms] delaying RPC due to: Service unavailable: Master config (127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [776ms] refreshing cache from master, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.61:42629, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.62:37111, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.60:43787, [779ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.61:42629: Network error: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629, [782ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.62:37111: Network error: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [784ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.60:43787: OK, [785ms] delaying RPC due to: Service unavailable: Master config (127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [1295ms] refreshing cache from master, [1296ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.61:42629, [1297ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.62:37111, [1298ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.61:42629: Network error: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629, [1300ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.60:43787, [1302ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.62:37111: Network error: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [1303ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.60:43787: OK, [1304ms] delaying RPC due to: Service unavailable: Master config (127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [1315ms] refreshing cache from master, [1315ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.61:42629, [1316ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.62:37111, [1317ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.60:43787, [1318ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.61:42629: Network error: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629, [1319ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.62:37111: Network error: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [1322ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.60:43787: OK, [1322ms] delaying RPC due to: Service unavailable: Master config (127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [2576ms] refreshing cache from master, [2576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.61:42629, [2577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.62:37111, [2578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.60:43787, [2580ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.61:42629: Network error: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629, [2584ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.62:37111: Network error: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [2585ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.60:43787: OK, [2586ms] delaying RPC due to: Service unavailable: Master config (127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [4416ms] refreshing cache from master, [4416ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.61:42629, [4417ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.62:37111, [4418ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.60:43787, [4419ms] too many traces: truncated at 100 traces, deferred=Deferred@889477129(state=PENDING, result=null, callback=(continuation of Deferred@1891718423 after retry RPC after error@1551332473) -> (continuation of Deferred@1319563932 after retry RPC after error@1509244753) -> (continuation of Deferred@593932437 after retry RPC after error@1154862140) -> (continuation of Deferred@1025438760 after retry RPC after error@1306610541) -> (continuation of Deferred@1536534620 after retry RPC after error@827013950) -> (continuation of Deferred@2067674763 after retry RPC after error@1011069090) -> (continuation of Deferred@1247946814 after retry RPC after error@580413575) -> (continuation of Deferred@672700465 after retry RPC after error@167766832) -> (continuation of Deferred@1376190953 after retry RPC after error@867862398) -> (continuation of Deferred@1594837032 after retry RPC after error@702516882) -> (continuation of Deferred@1136511497 after retry RPC after error@618345591) -> (continuation of Deferred@28337868 after retry RPC after error@1422836598) -> (continuation of Deferred@2039391844 after retry RPC after error@76238254) -> (continuation of Deferred@101703157 after retry RPC after error@620260145) -> (continuation of Deferred@684278860 after retry RPC after error@1995545286) -> (continuation of Deferred@250155416 after retry RPC after error@1764709015) -> (continuation of Deferred@195465434 after retry RPC after error@1912282306) -> (continuation of Deferred@936945332 after retry RPC after error@611381554) -> (continuation of Deferred@460297510 after retry RPC after error@853247877) -> (continuation of Deferred@1086618220 after retry RPC after error@1671359015) -> (continuation of Deferred@71275991 after retry RPC after error@1790365050) -> (continuation of Deferred@163765594 after retry RPC after error@488231319) -> (continuation of Deferred@1710259918 after retry RPC after error@963748375) -> (continuation of Deferred@650547310 after retry RPC after error@1962960968) -> (continuation of Deferred@325086760 after retry RPC after error@657790793) -> (continuation of Deferred@895661601 after retry RPC after error@1129765613), errback=(continuation of Deferred@1891718423 after retry RPC after error@1551332473) -> (continuation of Deferred@1319563932 after retry RPC after error@1509244753) -> (continuation of Deferred@593932437 after retry RPC after error@1154862140) -> (continuation of Deferred@1025438760 after retry RPC after error@1306610541) -> (continuation of Deferred@1536534620 after retry RPC after error@827013950) -> (continuation of Deferred@2067674763 after retry RPC after error@1011069090) -> (continuation of Deferred@1247946814 after retry RPC after error@580413575) -> (continuation of Deferred@672700465 after retry RPC after error@167766832) -> (continuation of Deferred@1376190953 after retry RPC after error@867862398) -> (continuation of Deferred@1594837032 after retry RPC after error@702516882) -> (continuation of Deferred@1136511497 after retry RPC after error@618345591) -> (continuation of Deferred@28337868 after retry RPC after error@1422836598) -> (continuation of Deferred@2039391844 after retry RPC after error@76238254) -> (continuation of Deferred@101703157 after retry RPC after error@620260145) -> (continuation of Deferred@684278860 after retry RPC after error@1995545286) -> (continuation of Deferred@250155416 after retry RPC after error@1764709015) -> (continuation of Deferred@195465434 after retry RPC after error@1912282306) -> (continuation of Deferred@936945332 after retry RPC after error@611381554) -> (continuation of Deferred@460297510 after retry RPC after error@853247877) -> (continuation of Deferred@1086618220 after retry RPC after error@1671359015) -> (continuation of Deferred@71275991 after retry RPC after error@1790365050) -> (continuation of Deferred@163765594 after retry RPC after error@488231319) -> (continuation of Deferred@1710259918 after retry RPC after error@963748375) -> (continuation of Deferred@650547310 after retry RPC after error@1962960968) -> (continuation of Deferred@325086760 after retry RPC after error@657790793) -> (continuation of Deferred@895661601 after retry RPC after error@1129765613)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.9.98.62:37111
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=29888), Traces: [0ms] refreshing cache from master, [30ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.61:42629, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.62:37111, [111ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.60:43787, [146ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.61:42629: Network error: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629, [145ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.62:37111: Network error: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [516ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.60:43787: OK, [520ms] delaying RPC due to: Service unavailable: Master config (127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629, [535ms] refreshing cache from master, [536ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.61:42629, [538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.62:37111, [539ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.60:43787, [543ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.61:42629: Network error: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629, [548ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.60:43787: OK, [550ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.62:37111: Network error: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [551ms] delaying RPC due to: Service unavailable: Master config (127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [555ms] refreshing cache from master, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.61:42629, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.62:37111, [558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.60:43787, [558ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.61:42629: Network error: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629, [561ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.62:37111: Network error: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [562ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.60:43787: OK, [564ms] delaying RPC due to: Service unavailable: Master config (127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [575ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.61:42629, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.62:37111, [578ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.61:42629: Network error: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629, [579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.60:43787, [584ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.60:43787: OK, [585ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.62:37111: Network error: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [586ms] delaying RPC due to: Service unavailable: Master config (127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [595ms] refreshing cache from master, [595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.61:42629, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.62:37111, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.60:43787, [599ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.61:42629: Network error: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629, [602ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.62:37111: Network error: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [605ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.60:43787: OK, [606ms] delaying RPC due to: Service unavailable: Master config (127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [636ms] refreshing cache from master, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.61:42629, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.62:37111, [639ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.60:43787, [639ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.61:42629: Network error: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629, [643ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.62:37111: Network error: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [645ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.60:43787: OK, [649ms] delaying RPC due to: Service unavailable: Master config (127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [695ms] refreshing cache from master, [695ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.61:42629, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.62:37111, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.60:43787, [699ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.61:42629: Network error: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629, [704ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.60:43787: OK, [705ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.62:37111: Network error: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [706ms] delaying RPC due to: Service unavailable: Master config (127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [736ms] refreshing cache from master, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.61:42629, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.62:37111, [739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.60:43787, [741ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.61:42629: Network error: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629, [742ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.62:37111: Network error: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [744ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.60:43787: OK, [745ms] delaying RPC due to: Service unavailable: Master config (127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [776ms] refreshing cache from master, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.61:42629, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.62:37111, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.60:43787, [779ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.61:42629: Network error: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629, [782ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.62:37111: Network error: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [784ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.60:43787: OK, [785ms] delaying RPC due to: Service unavailable: Master config (127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [1295ms] refreshing cache from master, [1296ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.61:42629, [1297ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.62:37111, [1298ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.61:42629: Network error: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629, [1300ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.60:43787, [1302ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.62:37111: Network error: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [1303ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.60:43787: OK, [1304ms] delaying RPC due to: Service unavailable: Master config (127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [1315ms] refreshing cache from master, [1315ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.61:42629, [1316ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.62:37111, [1317ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.60:43787, [1318ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.61:42629: Network error: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629, [1319ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.62:37111: Network error: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [1322ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.60:43787: OK, [1322ms] delaying RPC due to: Service unavailable: Master config (127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [2576ms] refreshing cache from master, [2576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.61:42629, [2577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.62:37111, [2578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.60:43787, [2580ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.61:42629: Network error: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629, [2584ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.62:37111: Network error: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [2585ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.60:43787: OK, [2586ms] delaying RPC due to: Service unavailable: Master config (127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [4416ms] refreshing cache from master, [4416ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.61:42629, [4417ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.62:37111, [4418ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.60:43787, [4419ms] too many traces: truncated at 100 traces, deferred=Deferred@889477129(state=PENDING, result=null, callback=(continuation of Deferred@1891718423 after retry RPC after error@1551332473) -> (continuation of Deferred@1319563932 after retry RPC after error@1509244753) -> (continuation of Deferred@593932437 after retry RPC after error@1154862140) -> (continuation of Deferred@1025438760 after retry RPC after error@1306610541) -> (continuation of Deferred@1536534620 after retry RPC after error@827013950) -> (continuation of Deferred@2067674763 after retry RPC after error@1011069090) -> (continuation of Deferred@1247946814 after retry RPC after error@580413575) -> (continuation of Deferred@672700465 after retry RPC after error@167766832) -> (continuation of Deferred@1376190953 after retry RPC after error@867862398) -> (continuation of Deferred@1594837032 after retry RPC after error@702516882) -> (continuation of Deferred@1136511497 after retry RPC after error@618345591) -> (continuation of Deferred@28337868 after retry RPC after error@1422836598) -> (continuation of Deferred@2039391844 after retry RPC after error@76238254) -> (continuation of Deferred@101703157 after retry RPC after error@620260145) -> (continuation of Deferred@684278860 after retry RPC after error@1995545286) -> (continuation of Deferred@250155416 after retry RPC after error@1764709015) -> (continuation of Deferred@195465434 after retry RPC after error@1912282306) -> (continuation of Deferred@936945332 after retry RPC after error@611381554) -> (continuation of Deferred@460297510 after retry RPC after error@853247877) -> (continuation of Deferred@1086618220 after retry RPC after error@1671359015) -> (continuation of Deferred@71275991 after retry RPC after error@1790365050) -> (continuation of Deferred@163765594 after retry RPC after error@488231319) -> (continuation of Deferred@1710259918 after retry RPC after error@963748375) -> (continuation of Deferred@650547310 after retry RPC after error@1962960968) -> (continuation of Deferred@325086760 after retry RPC after error@657790793) -> (continuation of Deferred@895661601 after retry RPC after error@1129765613), errback=(continuation of Deferred@1891718423 after retry RPC after error@1551332473) -> (continuation of Deferred@1319563932 after retry RPC after error@1509244753) -> (continuation of Deferred@593932437 after retry RPC after error@1154862140) -> (continuation of Deferred@1025438760 after retry RPC after error@1306610541) -> (continuation of Deferred@1536534620 after retry RPC after error@827013950) -> (continuation of Deferred@2067674763 after retry RPC after error@1011069090) -> (continuation of Deferred@1247946814 after retry RPC after error@580413575) -> (continuation of Deferred@672700465 after retry RPC after error@167766832) -> (continuation of Deferred@1376190953 after retry RPC after error@867862398) -> (continuation of Deferred@1594837032 after retry RPC after error@702516882) -> (continuation of Deferred@1136511497 after retry RPC after error@618345591) -> (continuation of Deferred@28337868 after retry RPC after error@1422836598) -> (continuation of Deferred@2039391844 after retry RPC after error@76238254) -> (continuation of Deferred@101703157 after retry RPC after error@620260145) -> (continuation of Deferred@684278860 after retry RPC after error@1995545286) -> (continuation of Deferred@250155416 after retry RPC after error@1764709015) -> (continuation of Deferred@195465434 after retry RPC after error@1912282306) -> (continuation of Deferred@936945332 after retry RPC after error@611381554) -> (continuation of Deferred@460297510 after retry RPC after error@853247877) -> (continuation of Deferred@1086618220 after retry RPC after error@1671359015) -> (continuation of Deferred@71275991 after retry RPC after error@1790365050) -> (continuation of Deferred@163765594 after retry RPC after error@488231319) -> (continuation of Deferred@1710259918 after retry RPC after error@963748375) -> (continuation of Deferred@650547310 after retry RPC after error@1962960968) -> (continuation of Deferred@325086760 after retry RPC after error@657790793) -> (continuation of Deferred@895661601 after retry RPC after error@1129765613)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.9.98.62:37111
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
10:28:50.280 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
10:28:50.280 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=29888), Traces: [0ms] refreshing cache from master, [30ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.61:42629, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.62:37111, [111ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.60:43787, [146ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.61:42629: Network error: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629, [145ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.62:37111: Network error: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [516ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.60:43787: OK, [520ms] delaying RPC due to: Service unavailable: Master config (127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629, [535ms] refreshing cache from master, [536ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.61:42629, [538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.62:37111, [539ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.60:43787, [543ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.61:42629: Network error: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629, [548ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.60:43787: OK, [550ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.62:37111: Network error: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [551ms] delaying RPC due to: Service unavailable: Master config (127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [555ms] refreshing cache from master, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.61:42629, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.62:37111, [558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.60:43787, [558ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.61:42629: Network error: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629, [561ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.62:37111: Network error: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [562ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.60:43787: OK, [564ms] delaying RPC due to: Service unavailable: Master config (127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [575ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.61:42629, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.62:37111, [578ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.61:42629: Network error: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629, [579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.60:43787, [584ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.60:43787: OK, [585ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.62:37111: Network error: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [586ms] delaying RPC due to: Service unavailable: Master config (127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [595ms] refreshing cache from master, [595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.61:42629, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.62:37111, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.60:43787, [599ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.61:42629: Network error: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629, [602ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.62:37111: Network error: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [605ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.60:43787: OK, [606ms] delaying RPC due to: Service unavailable: Master config (127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [636ms] refreshing cache from master, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.61:42629, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.62:37111, [639ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.60:43787, [639ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.61:42629: Network error: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629, [643ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.62:37111: Network error: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [645ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.60:43787: OK, [649ms] delaying RPC due to: Service unavailable: Master config (127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [695ms] refreshing cache from master, [695ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.61:42629, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.62:37111, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.60:43787, [699ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.61:42629: Network error: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629, [704ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.60:43787: OK, [705ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.62:37111: Network error: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [706ms] delaying RPC due to: Service unavailable: Master config (127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [736ms] refreshing cache from master, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.61:42629, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.62:37111, [739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.60:43787, [741ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.61:42629: Network error: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629, [742ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.62:37111: Network error: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [744ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.60:43787: OK, [745ms] delaying RPC due to: Service unavailable: Master config (127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [776ms] refreshing cache from master, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.61:42629, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.62:37111, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.60:43787, [779ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.61:42629: Network error: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629, [782ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.62:37111: Network error: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [784ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.60:43787: OK, [785ms] delaying RPC due to: Service unavailable: Master config (127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [1295ms] refreshing cache from master, [1296ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.61:42629, [1297ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.62:37111, [1298ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.61:42629: Network error: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629, [1300ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.60:43787, [1302ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.62:37111: Network error: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [1303ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.60:43787: OK, [1304ms] delaying RPC due to: Service unavailable: Master config (127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [1315ms] refreshing cache from master, [1315ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.61:42629, [1316ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.62:37111, [1317ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.60:43787, [1318ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.61:42629: Network error: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629, [1319ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.62:37111: Network error: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [1322ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.60:43787: OK, [1322ms] delaying RPC due to: Service unavailable: Master config (127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [2576ms] refreshing cache from master, [2576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.61:42629, [2577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.62:37111, [2578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.60:43787, [2580ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.61:42629: Network error: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629, [2584ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.62:37111: Network error: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [2585ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.60:43787: OK, [2586ms] delaying RPC due to: Service unavailable: Master config (127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [4416ms] refreshing cache from master, [4416ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.61:42629, [4417ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.62:37111, [4418ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.60:43787, [4419ms] too many traces: truncated at 100 traces, deferred=Deferred@889477129(state=PENDING, result=null, callback=(continuation of Deferred@1891718423 after retry RPC after error@1551332473) -> (continuation of Deferred@1319563932 after retry RPC after error@1509244753) -> (continuation of Deferred@593932437 after retry RPC after error@1154862140) -> (continuation of Deferred@1025438760 after retry RPC after error@1306610541) -> (continuation of Deferred@1536534620 after retry RPC after error@827013950) -> (continuation of Deferred@2067674763 after retry RPC after error@1011069090) -> (continuation of Deferred@1247946814 after retry RPC after error@580413575) -> (continuation of Deferred@672700465 after retry RPC after error@167766832) -> (continuation of Deferred@1376190953 after retry RPC after error@867862398) -> (continuation of Deferred@1594837032 after retry RPC after error@702516882) -> (continuation of Deferred@1136511497 after retry RPC after error@618345591) -> (continuation of Deferred@28337868 after retry RPC after error@1422836598) -> (continuation of Deferred@2039391844 after retry RPC after error@76238254) -> (continuation of Deferred@101703157 after retry RPC after error@620260145) -> (continuation of Deferred@684278860 after retry RPC after error@1995545286) -> (continuation of Deferred@250155416 after retry RPC after error@1764709015) -> (continuation of Deferred@195465434 after retry RPC after error@1912282306) -> (continuation of Deferred@936945332 after retry RPC after error@611381554) -> (continuation of Deferred@460297510 after retry RPC after error@853247877) -> (continuation of Deferred@1086618220 after retry RPC after error@1671359015) -> (continuation of Deferred@71275991 after retry RPC after error@1790365050) -> (continuation of Deferred@163765594 after retry RPC after error@488231319) -> (continuation of Deferred@1710259918 after retry RPC after error@963748375) -> (continuation of Deferred@650547310 after retry RPC after error@1962960968) -> (continuation of Deferred@325086760 after retry RPC after error@657790793) -> (continuation of Deferred@895661601 after retry RPC after error@1129765613), errback=(continuation of Deferred@1891718423 after retry RPC after error@1551332473) -> (continuation of Deferred@1319563932 after retry RPC after error@1509244753) -> (continuation of Deferred@593932437 after retry RPC after error@1154862140) -> (continuation of Deferred@1025438760 after retry RPC after error@1306610541) -> (continuation of Deferred@1536534620 after retry RPC after error@827013950) -> (continuation of Deferred@2067674763 after retry RPC after error@1011069090) -> (continuation of Deferred@1247946814 after retry RPC after error@580413575) -> (continuation of Deferred@672700465 after retry RPC after error@167766832) -> (continuation of Deferred@1376190953 after retry RPC after error@867862398) -> (continuation of Deferred@1594837032 after retry RPC after error@702516882) -> (continuation of Deferred@1136511497 after retry RPC after error@618345591) -> (continuation of Deferred@28337868 after retry RPC after error@1422836598) -> (continuation of Deferred@2039391844 after retry RPC after error@76238254) -> (continuation of Deferred@101703157 after retry RPC after error@620260145) -> (continuation of Deferred@684278860 after retry RPC after error@1995545286) -> (continuation of Deferred@250155416 after retry RPC after error@1764709015) -> (continuation of Deferred@195465434 after retry RPC after error@1912282306) -> (continuation of Deferred@936945332 after retry RPC after error@611381554) -> (continuation of Deferred@460297510 after retry RPC after error@853247877) -> (continuation of Deferred@1086618220 after retry RPC after error@1671359015) -> (continuation of Deferred@71275991 after retry RPC after error@1790365050) -> (continuation of Deferred@163765594 after retry RPC after error@488231319) -> (continuation of Deferred@1710259918 after retry RPC after error@963748375) -> (continuation of Deferred@650547310 after retry RPC after error@1962960968) -> (continuation of Deferred@325086760 after retry RPC after error@657790793) -> (continuation of Deferred@895661601 after retry RPC after error@1129765613)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.9.98.62:37111
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=29888), Traces: [0ms] refreshing cache from master, [30ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.61:42629, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.62:37111, [111ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.60:43787, [146ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.61:42629: Network error: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629, [145ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.62:37111: Network error: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [516ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.60:43787: OK, [520ms] delaying RPC due to: Service unavailable: Master config (127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629, [535ms] refreshing cache from master, [536ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.61:42629, [538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.62:37111, [539ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.60:43787, [543ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.61:42629: Network error: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629, [548ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.60:43787: OK, [550ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.62:37111: Network error: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [551ms] delaying RPC due to: Service unavailable: Master config (127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [555ms] refreshing cache from master, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.61:42629, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.62:37111, [558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.60:43787, [558ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.61:42629: Network error: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629, [561ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.62:37111: Network error: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [562ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.60:43787: OK, [564ms] delaying RPC due to: Service unavailable: Master config (127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [575ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.61:42629, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.62:37111, [578ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.61:42629: Network error: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629, [579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.60:43787, [584ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.60:43787: OK, [585ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.62:37111: Network error: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [586ms] delaying RPC due to: Service unavailable: Master config (127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [595ms] refreshing cache from master, [595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.61:42629, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.62:37111, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.60:43787, [599ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.61:42629: Network error: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629, [602ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.62:37111: Network error: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [605ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.60:43787: OK, [606ms] delaying RPC due to: Service unavailable: Master config (127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [636ms] refreshing cache from master, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.61:42629, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.62:37111, [639ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.60:43787, [639ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.61:42629: Network error: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629, [643ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.62:37111: Network error: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [645ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.60:43787: OK, [649ms] delaying RPC due to: Service unavailable: Master config (127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [695ms] refreshing cache from master, [695ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.61:42629, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.62:37111, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.60:43787, [699ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.61:42629: Network error: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629, [704ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.60:43787: OK, [705ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.62:37111: Network error: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [706ms] delaying RPC due to: Service unavailable: Master config (127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [736ms] refreshing cache from master, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.61:42629, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.62:37111, [739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.60:43787, [741ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.61:42629: Network error: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629, [742ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.62:37111: Network error: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [744ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.60:43787: OK, [745ms] delaying RPC due to: Service unavailable: Master config (127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [776ms] refreshing cache from master, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.61:42629, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.62:37111, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.60:43787, [779ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.61:42629: Network error: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629, [782ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.62:37111: Network error: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [784ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.60:43787: OK, [785ms] delaying RPC due to: Service unavailable: Master config (127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [1295ms] refreshing cache from master, [1296ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.61:42629, [1297ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.62:37111, [1298ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.61:42629: Network error: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629, [1300ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.60:43787, [1302ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.62:37111: Network error: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [1303ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.60:43787: OK, [1304ms] delaying RPC due to: Service unavailable: Master config (127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [1315ms] refreshing cache from master, [1315ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.61:42629, [1316ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.62:37111, [1317ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.60:43787, [1318ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.61:42629: Network error: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629, [1319ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.62:37111: Network error: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [1322ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.60:43787: OK, [1322ms] delaying RPC due to: Service unavailable: Master config (127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [2576ms] refreshing cache from master, [2576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.61:42629, [2577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.62:37111, [2578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.60:43787, [2580ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.61:42629: Network error: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629, [2584ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.62:37111: Network error: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [2585ms] Sub RPC ConnectToMaster: received response from server master-127.9.98.60:43787: OK, [2586ms] delaying RPC due to: Service unavailable: Master config (127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111, [4416ms] refreshing cache from master, [4416ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.61:42629, [4417ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.62:37111, [4418ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.98.60:43787, [4419ms] too many traces: truncated at 100 traces, deferred=Deferred@889477129(state=PENDING, result=null, callback=(continuation of Deferred@1891718423 after retry RPC after error@1551332473) -> (continuation of Deferred@1319563932 after retry RPC after error@1509244753) -> (continuation of Deferred@593932437 after retry RPC after error@1154862140) -> (continuation of Deferred@1025438760 after retry RPC after error@1306610541) -> (continuation of Deferred@1536534620 after retry RPC after error@827013950) -> (continuation of Deferred@2067674763 after retry RPC after error@1011069090) -> (continuation of Deferred@1247946814 after retry RPC after error@580413575) -> (continuation of Deferred@672700465 after retry RPC after error@167766832) -> (continuation of Deferred@1376190953 after retry RPC after error@867862398) -> (continuation of Deferred@1594837032 after retry RPC after error@702516882) -> (continuation of Deferred@1136511497 after retry RPC after error@618345591) -> (continuation of Deferred@28337868 after retry RPC after error@1422836598) -> (continuation of Deferred@2039391844 after retry RPC after error@76238254) -> (continuation of Deferred@101703157 after retry RPC after error@620260145) -> (continuation of Deferred@684278860 after retry RPC after error@1995545286) -> (continuation of Deferred@250155416 after retry RPC after error@1764709015) -> (continuation of Deferred@195465434 after retry RPC after error@1912282306) -> (continuation of Deferred@936945332 after retry RPC after error@611381554) -> (continuation of Deferred@460297510 after retry RPC after error@853247877) -> (continuation of Deferred@1086618220 after retry RPC after error@1671359015) -> (continuation of Deferred@71275991 after retry RPC after error@1790365050) -> (continuation of Deferred@163765594 after retry RPC after error@488231319) -> (continuation of Deferred@1710259918 after retry RPC after error@963748375) -> (continuation of Deferred@650547310 after retry RPC after error@1962960968) -> (continuation of Deferred@325086760 after retry RPC after error@657790793) -> (continuation of Deferred@895661601 after retry RPC after error@1129765613), errback=(continuation of Deferred@1891718423 after retry RPC after error@1551332473) -> (continuation of Deferred@1319563932 after retry RPC after error@1509244753) -> (continuation of Deferred@593932437 after retry RPC after error@1154862140) -> (continuation of Deferred@1025438760 after retry RPC after error@1306610541) -> (continuation of Deferred@1536534620 after retry RPC after error@827013950) -> (continuation of Deferred@2067674763 after retry RPC after error@1011069090) -> (continuation of Deferred@1247946814 after retry RPC after error@580413575) -> (continuation of Deferred@672700465 after retry RPC after error@167766832) -> (continuation of Deferred@1376190953 after retry RPC after error@867862398) -> (continuation of Deferred@1594837032 after retry RPC after error@702516882) -> (continuation of Deferred@1136511497 after retry RPC after error@618345591) -> (continuation of Deferred@28337868 after retry RPC after error@1422836598) -> (continuation of Deferred@2039391844 after retry RPC after error@76238254) -> (continuation of Deferred@101703157 after retry RPC after error@620260145) -> (continuation of Deferred@684278860 after retry RPC after error@1995545286) -> (continuation of Deferred@250155416 after retry RPC after error@1764709015) -> (continuation of Deferred@195465434 after retry RPC after error@1912282306) -> (continuation of Deferred@936945332 after retry RPC after error@611381554) -> (continuation of Deferred@460297510 after retry RPC after error@853247877) -> (continuation of Deferred@1086618220 after retry RPC after error@1671359015) -> (continuation of Deferred@71275991 after retry RPC after error@1790365050) -> (continuation of Deferred@163765594 after retry RPC after error@488231319) -> (continuation of Deferred@1710259918 after retry RPC after error@963748375) -> (continuation of Deferred@650547310 after retry RPC after error@1962960968) -> (continuation of Deferred@325086760 after retry RPC after error@657790793) -> (continuation of Deferred@895661601 after retry RPC after error@1129765613)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.9.98.61:42629,127.9.98.62:37111,127.9.98.60:43787) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.61:42629(127.9.98.61:42629): Connection refused: /127.9.98.61:42629,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.98.62:37111(127.9.98.62:37111): Connection refused: /127.9.98.62:37111
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.9.98.62:37111
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)