00:28:07.681 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
00:28:07.693 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskNE0Lyu/test-tmp
00:28:07.696 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
00:28:07.699 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskNE0Lyu/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
00:28:07.913 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
00:28:08.101 [DEBUG - main] (MiniKuduCluster.java:195) Response:
00:28:08.102 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
00:28:08.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:08.107404 7145 mini_oidc.cc:152] Starting JWKS server
00:28:08.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:08.111524 7145 webserver.cc:460] Webserver started at http://127.0.0.1:34577/ using document root <none> and password file <none>
00:28:08.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:08.112025 7145 mini_oidc.cc:189] Starting OIDC Discovery server
00:28:08.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:08.113837 7145 webserver.cc:460] Webserver started at http://127.0.0.1:38253/ using document root <none> and password file <none>
00:28:08.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-24T00:28:08Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
00:28:08.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-24T00:28:08Z Disabled control of system clock
00:28:08.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:08.164855 7145 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskNE0Lyu/build/tsan/bin/kudu
00:28:08.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNE0Lyu/build/tsan/bin/kudu
00:28:08.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/master-0/wal
00:28:08.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/master-0/data
00:28:08.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:28:08.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:28:08.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:28:08.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:08.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:08.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:08.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/master-0/logs
00:28:08.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/master-0/data/info.pb
00:28:08.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:08.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:28:08.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:28:08.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:28:08.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:28:08.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:08.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:08.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:08.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
00:28:08.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:28:08.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:28:08.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:28:08.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.250.126:46847
00:28:08.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.250.126
00:28:08.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:28:08.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.250.84:46577
00:28:08.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:08.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:08.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:28:08.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:28:08.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.250.126:46847,127.6.250.125:41277,127.6.250.124:32777
00:28:08.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:28:08.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34577/jwks.json with env {}
00:28:08.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:08.463528 7157 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
00:28:08.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:08.464140 7157 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:28:08.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:08.464799 7157 flags.cc:425] Enabled unsafe flag: --never_fsync=true
00:28:08.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:08.496129 7157 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
00:28:08.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:08.496403 7157 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
00:28:08.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:08.496591 7157 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
00:28:08.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:08.496809 7157 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
00:28:08.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:08.497080 7157 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
00:28:08.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:08.497309 7157 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34577/jwks.json
00:28:08.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:08.532042 7157 master_runner.cc:386] Master server non-default flags:
00:28:08.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:08.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.250.84:46577
00:28:08.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:08.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:28:08.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/master-0/data
00:28:08.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/master-0/wal
00:28:08.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:28:08.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.250.126:46847,127.6.250.125:41277,127.6.250.124:32777
00:28:08.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:08.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:08.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:28:08.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.250.126:46847
00:28:08.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:28:08.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:28:08.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:28:08.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34577/jwks.json
00:28:08.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:08.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:08.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/master-0/data/info.pb
00:28:08.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.250.126
00:28:08.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:28:08.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:28:08.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:08.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:28:08.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:28:08.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:08.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/master-0/logs
00:28:08.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:08.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:28:08.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
00:28:08.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
00:28:08.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
00:28:08.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
00:28:08.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:28:08.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 23 Sep 2026 23:32:10 UTC on ccycloud.quasar-wfxeao.root.comops.site
00:28:08.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4668917
00:28:08.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
00:28:08.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:08.533391 7157 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:28:08.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:08.535480 7157 file_cache.cc:493] Constructed file cache file cache with capacity 419430
00:28:08.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:08.555575 7166 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:09.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:08.555536 7164 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:09.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:08.555542 7163 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:09.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:09.754756 7165 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
00:28:09.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:09.754921 7157 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
00:28:09.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:09.760227 7157 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:28:09.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:09.763394 7157 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:28:09.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:09.764873 7157 hybrid_clock.cc:648] HybridClock initialized: now 1790209689764847 us; error 62 us; skew 500 ppm
00:28:09.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:09.765818 7157 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:28:09.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:09.785306 7157 webserver.cc:460] Webserver started at http://127.6.250.126:39143/ using document root <none> and password file <none>
00:28:09.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:09.786355 7157 fs_manager.cc:359] Metadata directory not provided
00:28:09.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:09.786550 7157 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:28:09.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:09.786962 7157 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
00:28:09.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:09.791932 7157 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/master-0/data/instance:
00:28:09.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "305bbef030aa4dbeb2c0bcd5718f0053"
00:28:09.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 00:28:09 on dist-test-slave-dnss"
00:28:09.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:09.793114 7157 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/master-0/wal/instance:
00:28:09.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "305bbef030aa4dbeb2c0bcd5718f0053"
00:28:09.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 00:28:09 on dist-test-slave-dnss"
00:28:09.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:09.800602 7157 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.007s sys 0.000s
00:28:09.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:09.806541 7173 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
00:28:09.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:09.808001 7157 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.000s
00:28:09.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:09.808403 7157 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/master-0/data,/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/master-0/wal
00:28:09.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "305bbef030aa4dbeb2c0bcd5718f0053"
00:28:09.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 00:28:09 on dist-test-slave-dnss"
00:28:09.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:09.808815 7157 fs_report.cc:383] FS layout report
00:28:09.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:28:09.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/master-0/wal
00:28:09.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/master-0/wal
00:28:09.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/master-0/data/data
00:28:09.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:28:09.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:28:09.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:28:09.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:28:09.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:28:09.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:28:09.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:28:09.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:28:09.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:28:09.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:28:09.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:28:09.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:09.846599 7157 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
00:28:09.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:09.848299 7157 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:28:09.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:09.848845 7157 kserver.cc:163] Server-wide thread pool size limit: 3276
00:28:09.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:09.920801 7157 rpc_server.cc:307] RPC server started. Bound to: 127.6.250.126:46847
00:28:09.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:09.920904 7224 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.250.126:46847 every 8 connection(s)
00:28:09.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:09.923686 7157 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/master-0/data/info.pb
00:28:09.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:09.929157 7225 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
00:28:09.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:09.929476 7145 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskNE0Lyu/build/tsan/bin/kudu as pid 7157
00:28:09.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:09.929991 7145 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/master-0/wal/instance
00:28:09.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:09.933481 7145 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskNE0Lyu/build/tsan/bin/kudu
00:28:09.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNE0Lyu/build/tsan/bin/kudu
00:28:09.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/master-1/wal
00:28:09.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/master-1/data
00:28:09.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:28:09.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:28:09.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:28:09.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:09.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:09.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:09.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/master-1/logs
00:28:09.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/master-1/data/info.pb
00:28:09.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:09.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:28:09.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:28:09.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:28:09.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:28:09.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:09.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:09.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:09.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
00:28:09.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:28:09.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:28:09.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:28:09.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.250.125:41277
00:28:09.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.250.125
00:28:09.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:28:09.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.250.84:46577
00:28:09.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:09.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:09.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:28:09.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:28:09.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.250.126:46847,127.6.250.125:41277,127.6.250.124:32777
00:28:09.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:28:09.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34577/jwks.json with env {}
00:28:09.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:09.945430 7225 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.250.126" port: 46847 } has no permanent_uuid. Determining permanent_uuid...
00:28:09.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:09.976962 7225 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.250.125" port: 41277 } has no permanent_uuid. Determining permanent_uuid...
00:28:09.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:09.979444 7177 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.250.125:41277: connect: Connection refused (error 111)
00:28:09.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:09.983716 7225 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.250.125:41277: Network error: Client connection negotiation failed: client connection to 127.6.250.125:41277: connect: Connection refused (error 111)
00:28:10.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:10.033197 7225 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.250.125" port: 41277 } attempt: 1
00:28:10.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:10.037433 7225 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.250.125:41277: Network error: Client connection negotiation failed: client connection to 127.6.250.125:41277: connect: Connection refused (error 111)
00:28:10.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:10.105912 7225 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.250.125" port: 41277 } attempt: 2
00:28:10.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:10.110575 7225 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.250.125:41277: Network error: Client connection negotiation failed: client connection to 127.6.250.125:41277: connect: Connection refused (error 111)
00:28:10.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:10.202055 7225 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.250.125" port: 41277 } attempt: 3
00:28:10.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:10.206219 7225 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.250.125:41277: Network error: Client connection negotiation failed: client connection to 127.6.250.125:41277: connect: Connection refused (error 111)
00:28:10.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:10.268693 7227 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
00:28:10.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:10.269423 7227 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:28:10.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:10.270107 7227 flags.cc:425] Enabled unsafe flag: --never_fsync=true
00:28:10.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:10.301828 7227 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
00:28:10.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:10.302227 7227 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
00:28:10.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:10.302554 7227 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
00:28:10.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:10.302960 7227 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
00:28:10.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:10.303313 7227 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
00:28:10.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:10.303567 7227 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34577/jwks.json
00:28:10.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:10.340418 7227 master_runner.cc:386] Master server non-default flags:
00:28:10.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:10.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.250.84:46577
00:28:10.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:10.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:28:10.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/master-1/data
00:28:10.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/master-1/wal
00:28:10.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:28:10.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.250.126:46847,127.6.250.125:41277,127.6.250.124:32777
00:28:10.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:10.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:10.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:28:10.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.250.125:41277
00:28:10.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:28:10.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:28:10.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:28:10.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34577/jwks.json
00:28:10.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:10.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:10.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/master-1/data/info.pb
00:28:10.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.250.125
00:28:10.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:28:10.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:28:10.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:10.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:28:10.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:28:10.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:10.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/master-1/logs
00:28:10.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:10.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:28:10.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
00:28:10.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
00:28:10.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
00:28:10.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
00:28:10.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:28:10.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 23 Sep 2026 23:32:10 UTC on ccycloud.quasar-wfxeao.root.comops.site
00:28:10.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4668917
00:28:10.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
00:28:10.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:10.342015 7227 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:28:10.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:10.344276 7227 file_cache.cc:493] Constructed file cache file cache with capacity 419430
00:28:10.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:10.349712 7225 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.250.125" port: 41277 } attempt: 4
00:28:10.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:10.353981 7225 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.250.125:41277: Network error: Client connection negotiation failed: client connection to 127.6.250.125:41277: connect: Connection refused (error 111)
00:28:10.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:10.364457 7237 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:10.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:10.653445 7225 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.250.125" port: 41277 } attempt: 5
00:28:10.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:10.657838 7225 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.250.125:41277: Network error: Client connection negotiation failed: client connection to 127.6.250.125:41277: connect: Connection refused (error 111)
00:28:11.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:11.205263 7225 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.250.125" port: 41277 } attempt: 6
00:28:11.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:11.216850 7225 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.250.125:41277: Network error: Client connection negotiation failed: client connection to 127.6.250.125:41277: connect: Connection refused (error 111)
00:28:11.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:10.364534 7236 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:11.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:10.364444 7239 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:11.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:10.366724 7227 server_base.cc:1034] running on GCE node
00:28:11.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:11.593940 7227 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:28:11.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:11.598270 7227 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:28:11.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:11.599785 7227 hybrid_clock.cc:648] HybridClock initialized: now 1790209691599637 us; error 157 us; skew 500 ppm
00:28:11.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:11.601131 7227 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:28:11.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:11.609254 7227 webserver.cc:460] Webserver started at http://127.6.250.125:45901/ using document root <none> and password file <none>
00:28:11.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:11.610258 7227 fs_manager.cc:359] Metadata directory not provided
00:28:11.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:11.610464 7227 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:28:11.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:11.610884 7227 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
00:28:11.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:11.616238 7227 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/master-1/data/instance:
00:28:11.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "791e8bcb122c4c3e98c5e01b5217df0f"
00:28:11.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 00:28:11 on dist-test-slave-dnss"
00:28:11.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:11.617493 7227 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/master-1/wal/instance:
00:28:11.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "791e8bcb122c4c3e98c5e01b5217df0f"
00:28:11.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 00:28:11 on dist-test-slave-dnss"
00:28:11.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:11.625221 7227 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.008s sys 0.000s
00:28:11.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:11.631384 7248 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
00:28:11.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:11.632491 7227 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
00:28:11.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:11.632850 7227 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/master-1/data,/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/master-1/wal
00:28:11.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "791e8bcb122c4c3e98c5e01b5217df0f"
00:28:11.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 00:28:11 on dist-test-slave-dnss"
00:28:11.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:11.633261 7227 fs_report.cc:383] FS layout report
00:28:11.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:28:11.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/master-1/wal
00:28:11.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/master-1/wal
00:28:11.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/master-1/data/data
00:28:11.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:28:11.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:28:11.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:28:11.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:28:11.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:28:11.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:28:11.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:28:11.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:28:11.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:28:11.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:28:11.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:28:11.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:11.700017 7227 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
00:28:11.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:11.701622 7227 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:28:11.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:11.702155 7227 kserver.cc:163] Server-wide thread pool size limit: 3276
00:28:11.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:11.774933 7227 rpc_server.cc:307] RPC server started. Bound to: 127.6.250.125:41277
00:28:11.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:11.775036 7299 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.250.125:41277 every 8 connection(s)
00:28:11.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:11.777745 7227 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/master-1/data/info.pb
00:28:11.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:11.782996 7300 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
00:28:11.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:11.784243 7145 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskNE0Lyu/build/tsan/bin/kudu as pid 7227
00:28:11.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:11.784569 7145 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/master-1/wal/instance
00:28:11.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:11.786458 7145 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskNE0Lyu/build/tsan/bin/kudu
00:28:11.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNE0Lyu/build/tsan/bin/kudu
00:28:11.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/master-2/wal
00:28:11.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/master-2/data
00:28:11.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:28:11.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:28:11.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:28:11.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:11.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:11.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:11.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/master-2/logs
00:28:11.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/master-2/data/info.pb
00:28:11.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:11.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:28:11.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:28:11.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:28:11.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:28:11.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:11.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:11.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:11.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
00:28:11.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:28:11.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:28:11.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:28:11.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.250.124:32777
00:28:11.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.250.124
00:28:11.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:28:11.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.250.84:46577
00:28:11.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:11.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:11.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:28:11.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:28:11.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.250.126:46847,127.6.250.125:41277,127.6.250.124:32777
00:28:11.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:28:11.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34577/jwks.json with env {}
00:28:11.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:11.798305 7300 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.250.126" port: 46847 } has no permanent_uuid. Determining permanent_uuid...
00:28:11.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:11.824381 7300 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.250.125" port: 41277 } has no permanent_uuid. Determining permanent_uuid...
00:28:11.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:11.841822 7300 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.250.124" port: 32777 } has no permanent_uuid. Determining permanent_uuid...
00:28:11.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:11.844261 7252 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.250.124:32777: connect: Connection refused (error 111)
00:28:11.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:11.848407 7300 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.250.124:32777: Network error: Client connection negotiation failed: client connection to 127.6.250.124:32777: connect: Connection refused (error 111)
00:28:11.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:11.898000 7300 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.250.124" port: 32777 } attempt: 1
00:28:11.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:11.901960 7300 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.250.124:32777: Network error: Client connection negotiation failed: client connection to 127.6.250.124:32777: connect: Connection refused (error 111)
00:28:11.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:11.970412 7300 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.250.124" port: 32777 } attempt: 2
00:28:11.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:11.973999 7300 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.250.124:32777: Network error: Client connection negotiation failed: client connection to 127.6.250.124:32777: connect: Connection refused (error 111)
00:28:12.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:12.065486 7300 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.250.124" port: 32777 } attempt: 3
00:28:12.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:12.069566 7300 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.250.124:32777: Network error: Client connection negotiation failed: client connection to 127.6.250.124:32777: connect: Connection refused (error 111)
00:28:12.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:12.113809 7302 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
00:28:12.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:12.114375 7302 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:28:12.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:12.114936 7302 flags.cc:425] Enabled unsafe flag: --never_fsync=true
00:28:12.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:12.146287 7302 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
00:28:12.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:12.146600 7302 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
00:28:12.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:12.146842 7302 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
00:28:12.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:12.147025 7302 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
00:28:12.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:12.147197 7302 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
00:28:12.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:12.147328 7302 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34577/jwks.json
00:28:12.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:12.185424 7302 master_runner.cc:386] Master server non-default flags:
00:28:12.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:12.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.250.84:46577
00:28:12.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:12.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:28:12.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/master-2/data
00:28:12.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/master-2/wal
00:28:12.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:28:12.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.250.126:46847,127.6.250.125:41277,127.6.250.124:32777
00:28:12.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:12.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:12.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:28:12.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.250.124:32777
00:28:12.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:28:12.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:28:12.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:28:12.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34577/jwks.json
00:28:12.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:12.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:12.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/master-2/data/info.pb
00:28:12.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.250.124
00:28:12.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:28:12.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:28:12.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:12.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:28:12.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:28:12.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:12.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/master-2/logs
00:28:12.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:12.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:28:12.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
00:28:12.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
00:28:12.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
00:28:12.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
00:28:12.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:28:12.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 23 Sep 2026 23:32:10 UTC on ccycloud.quasar-wfxeao.root.comops.site
00:28:12.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4668917
00:28:12.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
00:28:12.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:12.186856 7302 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:28:12.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:12.189003 7302 file_cache.cc:493] Constructed file cache file cache with capacity 419430
00:28:12.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:12.208395 7312 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:12.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:12.213060 7300 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.250.124" port: 32777 } attempt: 4
00:28:12.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:12.217126 7300 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.250.124:32777: Network error: Client connection negotiation failed: client connection to 127.6.250.124:32777: connect: Connection refused (error 111)
00:28:12.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:12.277331 7225 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.250.125" port: 41277 } attempt: 7
00:28:12.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:12.293490 7225 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.250.124" port: 32777 } has no permanent_uuid. Determining permanent_uuid...
00:28:12.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:12.298837 7225 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.250.124:32777: Network error: Client connection negotiation failed: client connection to 127.6.250.124:32777: connect: Connection refused (error 111)
00:28:12.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:12.357466 7225 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.250.124" port: 32777 } attempt: 1
00:28:12.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:12.362484 7225 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.250.124:32777: Network error: Client connection negotiation failed: client connection to 127.6.250.124:32777: connect: Connection refused (error 111)
00:28:12.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:12.443922 7225 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.250.124" port: 32777 } attempt: 2
00:28:12.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:12.448462 7225 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.250.124:32777: Network error: Client connection negotiation failed: client connection to 127.6.250.124:32777: connect: Connection refused (error 111)
00:28:12.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:12.516518 7300 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.250.124" port: 32777 } attempt: 5
00:28:12.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:12.520902 7300 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.250.124:32777: Network error: Client connection negotiation failed: client connection to 127.6.250.124:32777: connect: Connection refused (error 111)
00:28:12.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:12.533949 7225 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.250.124" port: 32777 } attempt: 3
00:28:12.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:12.538131 7225 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.250.124:32777: Network error: Client connection negotiation failed: client connection to 127.6.250.124:32777: connect: Connection refused (error 111)
00:28:12.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:12.678553 7225 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.250.124" port: 32777 } attempt: 4
00:28:12.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:12.683554 7225 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.250.124:32777: Network error: Client connection negotiation failed: client connection to 127.6.250.124:32777: connect: Connection refused (error 111)
00:28:12.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:12.967074 7225 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.250.124" port: 32777 } attempt: 5
00:28:12.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:12.971887 7225 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.250.124:32777: Network error: Client connection negotiation failed: client connection to 127.6.250.124:32777: connect: Connection refused (error 111)
00:28:13.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:13.068295 7300 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.250.124" port: 32777 } attempt: 6
00:28:13.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:13.076050 7300 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.250.124:32777: Network error: Client connection negotiation failed: client connection to 127.6.250.124:32777: connect: Connection refused (error 111)
00:28:13.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:12.208489 7313 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:13.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:12.208717 7315 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:13.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:12.211282 7302 server_base.cc:1034] running on GCE node
00:28:13.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:13.464676 7302 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:28:13.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:13.467897 7302 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:28:13.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:13.469372 7302 hybrid_clock.cc:648] HybridClock initialized: now 1790209693469336 us; error 59 us; skew 500 ppm
00:28:13.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:13.470232 7302 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:28:13.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:13.477658 7302 webserver.cc:460] Webserver started at http://127.6.250.124:42699/ using document root <none> and password file <none>
00:28:13.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:13.478585 7302 fs_manager.cc:359] Metadata directory not provided
00:28:13.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:13.478778 7302 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:28:13.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:13.479212 7302 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
00:28:13.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:13.486701 7302 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/master-2/data/instance:
00:28:13.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "739522ae64834cf0b7591777d2ccbd28"
00:28:13.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 00:28:13 on dist-test-slave-dnss"
00:28:13.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:13.488389 7302 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/master-2/wal/instance:
00:28:13.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "739522ae64834cf0b7591777d2ccbd28"
00:28:13.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 00:28:13 on dist-test-slave-dnss"
00:28:13.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:13.498413 7302 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.009s sys 0.000s
00:28:13.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:13.505339 7326 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
00:28:13.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:13.506620 7302 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
00:28:13.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:13.506938 7302 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/master-2/data,/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/master-2/wal
00:28:13.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "739522ae64834cf0b7591777d2ccbd28"
00:28:13.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 00:28:13 on dist-test-slave-dnss"
00:28:13.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:13.507301 7302 fs_report.cc:383] FS layout report
00:28:13.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:28:13.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/master-2/wal
00:28:13.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/master-2/wal
00:28:13.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/master-2/data/data
00:28:13.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:28:13.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:28:13.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:28:13.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:28:13.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:28:13.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:28:13.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:28:13.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:28:13.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:28:13.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:28:13.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:28:13.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:13.524510 7225 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.250.124" port: 32777 } attempt: 6
00:28:13.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:13.533392 7225 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.250.124:32777: Network error: Client connection negotiation failed: client connection to 127.6.250.124:32777: connect: Connection refused (error 111)
00:28:13.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:13.552263 7302 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
00:28:13.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:13.553860 7302 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:28:13.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:13.554339 7302 kserver.cc:163] Server-wide thread pool size limit: 3276
00:28:13.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:13.631412 7302 rpc_server.cc:307] RPC server started. Bound to: 127.6.250.124:32777
00:28:13.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:13.631528 7379 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.250.124:32777 every 8 connection(s)
00:28:13.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:13.634310 7302 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/master-2/data/info.pb
00:28:13.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:13.639899 7380 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
00:28:13.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:13.640807 7145 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskNE0Lyu/build/tsan/bin/kudu as pid 7302
00:28:13.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:13.641219 7145 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/master-2/wal/instance
00:28:13.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:13.654414 7380 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.250.126" port: 46847 } has no permanent_uuid. Determining permanent_uuid...
00:28:13.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:13.687987 7380 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.250.125" port: 41277 } has no permanent_uuid. Determining permanent_uuid...
00:28:13.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:13.700392 7380 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.250.124" port: 32777 } has no permanent_uuid. Determining permanent_uuid...
00:28:13.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:13.717444 7380 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 739522ae64834cf0b7591777d2ccbd28: Bootstrap starting.
00:28:13.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:13.723870 7380 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 739522ae64834cf0b7591777d2ccbd28: Neither blocks nor log segments found. Creating new log.
00:28:13.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:13.726073 7380 log.cc:824] T 00000000000000000000000000000000 P 739522ae64834cf0b7591777d2ccbd28: Log is configured to *not* fsync() on all Append() calls
00:28:13.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:13.732638 7380 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 739522ae64834cf0b7591777d2ccbd28: No bootstrap required, opened a new log
00:28:13.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:13.757421 7380 raft_consensus.cc:348] T 00000000000000000000000000000000 P 739522ae64834cf0b7591777d2ccbd28 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "305bbef030aa4dbeb2c0bcd5718f0053" member_type: VOTER last_known_addr { host: "127.6.250.126" port: 46847 } } peers { permanent_uuid: "791e8bcb122c4c3e98c5e01b5217df0f" member_type: VOTER last_known_addr { host: "127.6.250.125" port: 41277 } } peers { permanent_uuid: "739522ae64834cf0b7591777d2ccbd28" member_type: VOTER last_known_addr { host: "127.6.250.124" port: 32777 } }
00:28:13.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:13.758024 7380 raft_consensus.cc:374] T 00000000000000000000000000000000 P 739522ae64834cf0b7591777d2ccbd28 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
00:28:13.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:13.758208 7380 raft_consensus.cc:729] T 00000000000000000000000000000000 P 739522ae64834cf0b7591777d2ccbd28 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 739522ae64834cf0b7591777d2ccbd28, State: Initialized, Role: FOLLOWER
00:28:13.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:13.759119 7380 consensus_queue.cc:260] T 00000000000000000000000000000000 P 739522ae64834cf0b7591777d2ccbd28 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "305bbef030aa4dbeb2c0bcd5718f0053" member_type: VOTER last_known_addr { host: "127.6.250.126" port: 46847 } } peers { permanent_uuid: "791e8bcb122c4c3e98c5e01b5217df0f" member_type: VOTER last_known_addr { host: "127.6.250.125" port: 41277 } } peers { permanent_uuid: "739522ae64834cf0b7591777d2ccbd28" member_type: VOTER last_known_addr { host: "127.6.250.124" port: 32777 } }
00:28:13.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:13.761809 7389 sys_catalog.cc:455] T 00000000000000000000000000000000 P 739522ae64834cf0b7591777d2ccbd28 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "305bbef030aa4dbeb2c0bcd5718f0053" member_type: VOTER last_known_addr { host: "127.6.250.126" port: 46847 } } peers { permanent_uuid: "791e8bcb122c4c3e98c5e01b5217df0f" member_type: VOTER last_known_addr { host: "127.6.250.125" port: 41277 } } peers { permanent_uuid: "739522ae64834cf0b7591777d2ccbd28" member_type: VOTER last_known_addr { host: "127.6.250.124" port: 32777 } } }
00:28:13.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:13.762974 7389 sys_catalog.cc:458] T 00000000000000000000000000000000 P 739522ae64834cf0b7591777d2ccbd28 [sys.catalog]: This master's current role is: FOLLOWER
00:28:13.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:13.763389 7380 sys_catalog.cc:564] T 00000000000000000000000000000000 P 739522ae64834cf0b7591777d2ccbd28 [sys.catalog]: configured and running, proceeding with master startup.
00:28:13.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:13.792158 7400 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 739522ae64834cf0b7591777d2ccbd28: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:28:13.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:13.792943 7400 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:28:13.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:13.810837 7334 tablet.cc:2369] T 00000000000000000000000000000000 P 739522ae64834cf0b7591777d2ccbd28: Can't schedule compaction. Clean time has not been advanced past its initial value.
00:28:13.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:13.787876 7302 master_runner.cc:186] Error getting master registration for 127.6.250.126:46847: OK, instance_id { permanent_uuid: "305bbef030aa4dbeb2c0bcd5718f0053" instance_seqno: 1790209689901104 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
00:28:13.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:13.817916 7302 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
00:28:14.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:14.136492 7300 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.250.124" port: 32777 } attempt: 7
00:28:14.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:14.159484 7300 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 791e8bcb122c4c3e98c5e01b5217df0f: Bootstrap starting.
00:28:14.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:14.166771 7300 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 791e8bcb122c4c3e98c5e01b5217df0f: Neither blocks nor log segments found. Creating new log.
00:28:14.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:14.169131 7300 log.cc:824] T 00000000000000000000000000000000 P 791e8bcb122c4c3e98c5e01b5217df0f: Log is configured to *not* fsync() on all Append() calls
00:28:14.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:14.174073 7300 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 791e8bcb122c4c3e98c5e01b5217df0f: No bootstrap required, opened a new log
00:28:14.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:14.206173 7300 raft_consensus.cc:348] T 00000000000000000000000000000000 P 791e8bcb122c4c3e98c5e01b5217df0f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "305bbef030aa4dbeb2c0bcd5718f0053" member_type: VOTER last_known_addr { host: "127.6.250.126" port: 46847 } } peers { permanent_uuid: "791e8bcb122c4c3e98c5e01b5217df0f" member_type: VOTER last_known_addr { host: "127.6.250.125" port: 41277 } } peers { permanent_uuid: "739522ae64834cf0b7591777d2ccbd28" member_type: VOTER last_known_addr { host: "127.6.250.124" port: 32777 } }
00:28:14.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:14.207438 7300 raft_consensus.cc:374] T 00000000000000000000000000000000 P 791e8bcb122c4c3e98c5e01b5217df0f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
00:28:14.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:14.208683 7300 raft_consensus.cc:729] T 00000000000000000000000000000000 P 791e8bcb122c4c3e98c5e01b5217df0f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 791e8bcb122c4c3e98c5e01b5217df0f, State: Initialized, Role: FOLLOWER
00:28:14.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:14.209739 7300 consensus_queue.cc:260] T 00000000000000000000000000000000 P 791e8bcb122c4c3e98c5e01b5217df0f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "305bbef030aa4dbeb2c0bcd5718f0053" member_type: VOTER last_known_addr { host: "127.6.250.126" port: 46847 } } peers { permanent_uuid: "791e8bcb122c4c3e98c5e01b5217df0f" member_type: VOTER last_known_addr { host: "127.6.250.125" port: 41277 } } peers { permanent_uuid: "739522ae64834cf0b7591777d2ccbd28" member_type: VOTER last_known_addr { host: "127.6.250.124" port: 32777 } }
00:28:14.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:14.212498 7404 sys_catalog.cc:455] T 00000000000000000000000000000000 P 791e8bcb122c4c3e98c5e01b5217df0f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "305bbef030aa4dbeb2c0bcd5718f0053" member_type: VOTER last_known_addr { host: "127.6.250.126" port: 46847 } } peers { permanent_uuid: "791e8bcb122c4c3e98c5e01b5217df0f" member_type: VOTER last_known_addr { host: "127.6.250.125" port: 41277 } } peers { permanent_uuid: "739522ae64834cf0b7591777d2ccbd28" member_type: VOTER last_known_addr { host: "127.6.250.124" port: 32777 } } }
00:28:14.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:14.213281 7404 sys_catalog.cc:458] T 00000000000000000000000000000000 P 791e8bcb122c4c3e98c5e01b5217df0f [sys.catalog]: This master's current role is: FOLLOWER
00:28:14.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:14.213953 7300 sys_catalog.cc:564] T 00000000000000000000000000000000 P 791e8bcb122c4c3e98c5e01b5217df0f [sys.catalog]: configured and running, proceeding with master startup.
00:28:14.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:14.242043 7389 raft_consensus.cc:482] T 00000000000000000000000000000000 P 739522ae64834cf0b7591777d2ccbd28 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
00:28:14.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:14.242570 7389 raft_consensus.cc:504] T 00000000000000000000000000000000 P 739522ae64834cf0b7591777d2ccbd28 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "305bbef030aa4dbeb2c0bcd5718f0053" member_type: VOTER last_known_addr { host: "127.6.250.126" port: 46847 } } peers { permanent_uuid: "791e8bcb122c4c3e98c5e01b5217df0f" member_type: VOTER last_known_addr { host: "127.6.250.125" port: 41277 } } peers { permanent_uuid: "739522ae64834cf0b7591777d2ccbd28" member_type: VOTER last_known_addr { host: "127.6.250.124" port: 32777 } }
00:28:14.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:14.245434 7389 leader_election.cc:288] T 00000000000000000000000000000000 P 739522ae64834cf0b7591777d2ccbd28 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 305bbef030aa4dbeb2c0bcd5718f0053 (127.6.250.126:46847), 791e8bcb122c4c3e98c5e01b5217df0f (127.6.250.125:41277)
00:28:14.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:14.247334 7275 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "739522ae64834cf0b7591777d2ccbd28" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "791e8bcb122c4c3e98c5e01b5217df0f" is_pre_election: true
00:28:14.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:14.248296 7275 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 791e8bcb122c4c3e98c5e01b5217df0f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 739522ae64834cf0b7591777d2ccbd28 in term 0.
00:28:14.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:14.250058 7332 leader_election.cc:302] T 00000000000000000000000000000000 P 739522ae64834cf0b7591777d2ccbd28 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 739522ae64834cf0b7591777d2ccbd28, 791e8bcb122c4c3e98c5e01b5217df0f; no voters:
00:28:14.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:14.251329 7389 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 739522ae64834cf0b7591777d2ccbd28 [term 0 FOLLOWER]: Leader pre-election won for term 1
00:28:14.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:14.251613 7415 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 791e8bcb122c4c3e98c5e01b5217df0f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:28:14.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:14.252168 7415 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:28:14.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:14.252450 7389 raft_consensus.cc:482] T 00000000000000000000000000000000 P 739522ae64834cf0b7591777d2ccbd28 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
00:28:14.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:14.252789 7389 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 739522ae64834cf0b7591777d2ccbd28 [term 0 FOLLOWER]: Advancing to term 1
00:28:14.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:14.259418 7389 raft_consensus.cc:504] T 00000000000000000000000000000000 P 739522ae64834cf0b7591777d2ccbd28 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "305bbef030aa4dbeb2c0bcd5718f0053" member_type: VOTER last_known_addr { host: "127.6.250.126" port: 46847 } } peers { permanent_uuid: "791e8bcb122c4c3e98c5e01b5217df0f" member_type: VOTER last_known_addr { host: "127.6.250.125" port: 41277 } } peers { permanent_uuid: "739522ae64834cf0b7591777d2ccbd28" member_type: VOTER last_known_addr { host: "127.6.250.124" port: 32777 } }
00:28:14.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:14.263586 7275 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "739522ae64834cf0b7591777d2ccbd28" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "791e8bcb122c4c3e98c5e01b5217df0f"
00:28:14.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:14.264170 7275 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 791e8bcb122c4c3e98c5e01b5217df0f [term 0 FOLLOWER]: Advancing to term 1
00:28:14.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:14.271209 7275 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 791e8bcb122c4c3e98c5e01b5217df0f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 739522ae64834cf0b7591777d2ccbd28 in term 1.
00:28:14.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:14.272536 7332 leader_election.cc:302] T 00000000000000000000000000000000 P 739522ae64834cf0b7591777d2ccbd28 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 739522ae64834cf0b7591777d2ccbd28, 791e8bcb122c4c3e98c5e01b5217df0f; no voters:
00:28:14.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:14.279008 7389 leader_election.cc:288] T 00000000000000000000000000000000 P 739522ae64834cf0b7591777d2ccbd28 [CANDIDATE]: Term 1 election: Requested vote from peers 305bbef030aa4dbeb2c0bcd5718f0053 (127.6.250.126:46847), 791e8bcb122c4c3e98c5e01b5217df0f (127.6.250.125:41277)
00:28:14.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:14.237517 7227 master_runner.cc:186] Error getting master registration for 127.6.250.126:46847: OK, instance_id { permanent_uuid: "305bbef030aa4dbeb2c0bcd5718f0053" instance_seqno: 1790209689901104 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
00:28:14.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:14.287081 7227 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
00:28:14.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:14.289067 7389 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 739522ae64834cf0b7591777d2ccbd28 [term 1 FOLLOWER]: Leader election won for term 1
00:28:14.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:14.289562 7389 raft_consensus.cc:686] T 00000000000000000000000000000000 P 739522ae64834cf0b7591777d2ccbd28 [term 1 LEADER]: Becoming Leader. State: Replica: 739522ae64834cf0b7591777d2ccbd28, State: Running, Role: LEADER
00:28:14.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:14.290481 7389 consensus_queue.cc:237] T 00000000000000000000000000000000 P 739522ae64834cf0b7591777d2ccbd28 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "305bbef030aa4dbeb2c0bcd5718f0053" member_type: VOTER last_known_addr { host: "127.6.250.126" port: 46847 } } peers { permanent_uuid: "791e8bcb122c4c3e98c5e01b5217df0f" member_type: VOTER last_known_addr { host: "127.6.250.125" port: 41277 } } peers { permanent_uuid: "739522ae64834cf0b7591777d2ccbd28" member_type: VOTER last_known_addr { host: "127.6.250.124" port: 32777 } }
00:28:14.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:14.311573 7417 sys_catalog.cc:455] T 00000000000000000000000000000000 P 739522ae64834cf0b7591777d2ccbd28 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 739522ae64834cf0b7591777d2ccbd28. Latest consensus state: current_term: 1 leader_uuid: "739522ae64834cf0b7591777d2ccbd28" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "305bbef030aa4dbeb2c0bcd5718f0053" member_type: VOTER last_known_addr { host: "127.6.250.126" port: 46847 } } peers { permanent_uuid: "791e8bcb122c4c3e98c5e01b5217df0f" member_type: VOTER last_known_addr { host: "127.6.250.125" port: 41277 } } peers { permanent_uuid: "739522ae64834cf0b7591777d2ccbd28" member_type: VOTER last_known_addr { host: "127.6.250.124" port: 32777 } } }
00:28:14.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:14.312706 7417 sys_catalog.cc:458] T 00000000000000000000000000000000 P 739522ae64834cf0b7591777d2ccbd28 [sys.catalog]: This master's current role is: LEADER
00:28:14.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:14.316035 7419 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
00:28:14.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:14.322954 7419 catalog_manager.cc:1484] Initializing Kudu cluster ID...
00:28:14.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:14.340430 7275 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 791e8bcb122c4c3e98c5e01b5217df0f [term 1 FOLLOWER]: Refusing update from remote peer 739522ae64834cf0b7591777d2ccbd28: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
00:28:14.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:14.342100 7417 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 739522ae64834cf0b7591777d2ccbd28 [LEADER]: Connected to new peer: Peer: permanent_uuid: "791e8bcb122c4c3e98c5e01b5217df0f" member_type: VOTER last_known_addr { host: "127.6.250.125" port: 41277 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
00:28:14.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:14.370123 7404 sys_catalog.cc:455] T 00000000000000000000000000000000 P 791e8bcb122c4c3e98c5e01b5217df0f [sys.catalog]: SysCatalogTable state changed. Reason: New leader 739522ae64834cf0b7591777d2ccbd28. Latest consensus state: current_term: 1 leader_uuid: "739522ae64834cf0b7591777d2ccbd28" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "305bbef030aa4dbeb2c0bcd5718f0053" member_type: VOTER last_known_addr { host: "127.6.250.126" port: 46847 } } peers { permanent_uuid: "791e8bcb122c4c3e98c5e01b5217df0f" member_type: VOTER last_known_addr { host: "127.6.250.125" port: 41277 } } peers { permanent_uuid: "739522ae64834cf0b7591777d2ccbd28" member_type: VOTER last_known_addr { host: "127.6.250.124" port: 32777 } } }
00:28:14.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:14.371289 7404 sys_catalog.cc:458] T 00000000000000000000000000000000 P 791e8bcb122c4c3e98c5e01b5217df0f [sys.catalog]: This master's current role is: FOLLOWER
00:28:14.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:14.375500 7420 mvcc.cc:205] Tried to move back new op lower bound from 7332698908002287616 to 7332698907823415296. Current Snapshot: MvccSnapshot[applied={T|T < 7332698908002287616}]
00:28:14.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:14.381165 7389 sys_catalog.cc:455] T 00000000000000000000000000000000 P 739522ae64834cf0b7591777d2ccbd28 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "739522ae64834cf0b7591777d2ccbd28" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "305bbef030aa4dbeb2c0bcd5718f0053" member_type: VOTER last_known_addr { host: "127.6.250.126" port: 46847 } } peers { permanent_uuid: "791e8bcb122c4c3e98c5e01b5217df0f" member_type: VOTER last_known_addr { host: "127.6.250.125" port: 41277 } } peers { permanent_uuid: "739522ae64834cf0b7591777d2ccbd28" member_type: VOTER last_known_addr { host: "127.6.250.124" port: 32777 } } }
00:28:14.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:14.381964 7389 sys_catalog.cc:458] T 00000000000000000000000000000000 P 739522ae64834cf0b7591777d2ccbd28 [sys.catalog]: This master's current role is: LEADER
00:28:14.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:14.394284 7404 sys_catalog.cc:455] T 00000000000000000000000000000000 P 791e8bcb122c4c3e98c5e01b5217df0f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "739522ae64834cf0b7591777d2ccbd28" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "305bbef030aa4dbeb2c0bcd5718f0053" member_type: VOTER last_known_addr { host: "127.6.250.126" port: 46847 } } peers { permanent_uuid: "791e8bcb122c4c3e98c5e01b5217df0f" member_type: VOTER last_known_addr { host: "127.6.250.125" port: 41277 } } peers { permanent_uuid: "739522ae64834cf0b7591777d2ccbd28" member_type: VOTER last_known_addr { host: "127.6.250.124" port: 32777 } } }
00:28:14.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:14.395179 7404 sys_catalog.cc:458] T 00000000000000000000000000000000 P 791e8bcb122c4c3e98c5e01b5217df0f [sys.catalog]: This master's current role is: FOLLOWER
00:28:14.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:14.395963 7419 catalog_manager.cc:1347] Generated new cluster ID: 3346e965890b4475b56752dccda948ea
00:28:14.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:14.396236 7419 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
00:28:14.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:14.438796 7419 catalog_manager.cc:1370] Generated new certificate authority record
00:28:14.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:14.441609 7419 catalog_manager.cc:1504] Loading token signing keys...
00:28:14.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:14.464908 7419 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 739522ae64834cf0b7591777d2ccbd28: Generated new TSK 0
00:28:14.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:14.466795 7419 catalog_manager.cc:1514] Initializing in-progress tserver states...
00:28:14.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:14.482419 7145 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskNE0Lyu/build/tsan/bin/kudu
00:28:14.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNE0Lyu/build/tsan/bin/kudu
00:28:14.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/ts-0/wal
00:28:14.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/ts-0/data
00:28:14.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:28:14.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:28:14.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:28:14.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:14.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:14.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:14.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/ts-0/logs
00:28:14.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/ts-0/data/info.pb
00:28:14.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:14.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:28:14.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:28:14.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:28:14.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:28:14.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:14.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:14.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:14.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:28:14.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:28:14.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.250.65:0
00:28:14.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.250.65
00:28:14.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.250.65
00:28:14.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:28:14.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.250.126:46847,127.6.250.125:41277,127.6.250.124:32777
00:28:14.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.250.84:46577
00:28:14.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:14.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:14.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:28:14.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:14.566798 7225 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.250.124" port: 32777 } attempt: 7
00:28:14.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:14.619289 7225 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 305bbef030aa4dbeb2c0bcd5718f0053: Bootstrap starting.
00:28:14.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:14.676165 7225 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 305bbef030aa4dbeb2c0bcd5718f0053: Neither blocks nor log segments found. Creating new log.
00:28:14.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:14.691310 7225 log.cc:824] T 00000000000000000000000000000000 P 305bbef030aa4dbeb2c0bcd5718f0053: Log is configured to *not* fsync() on all Append() calls
00:28:14.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:14.732439 7225 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 305bbef030aa4dbeb2c0bcd5718f0053: No bootstrap required, opened a new log
00:28:15.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:15.050510 7225 raft_consensus.cc:348] T 00000000000000000000000000000000 P 305bbef030aa4dbeb2c0bcd5718f0053 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "305bbef030aa4dbeb2c0bcd5718f0053" member_type: VOTER last_known_addr { host: "127.6.250.126" port: 46847 } } peers { permanent_uuid: "791e8bcb122c4c3e98c5e01b5217df0f" member_type: VOTER last_known_addr { host: "127.6.250.125" port: 41277 } } peers { permanent_uuid: "739522ae64834cf0b7591777d2ccbd28" member_type: VOTER last_known_addr { host: "127.6.250.124" port: 32777 } }
00:28:15.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:15.051436 7225 raft_consensus.cc:374] T 00000000000000000000000000000000 P 305bbef030aa4dbeb2c0bcd5718f0053 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
00:28:15.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:15.051743 7225 raft_consensus.cc:729] T 00000000000000000000000000000000 P 305bbef030aa4dbeb2c0bcd5718f0053 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 305bbef030aa4dbeb2c0bcd5718f0053, State: Initialized, Role: FOLLOWER
00:28:15.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:15.052681 7225 consensus_queue.cc:260] T 00000000000000000000000000000000 P 305bbef030aa4dbeb2c0bcd5718f0053 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "305bbef030aa4dbeb2c0bcd5718f0053" member_type: VOTER last_known_addr { host: "127.6.250.126" port: 46847 } } peers { permanent_uuid: "791e8bcb122c4c3e98c5e01b5217df0f" member_type: VOTER last_known_addr { host: "127.6.250.125" port: 41277 } } peers { permanent_uuid: "739522ae64834cf0b7591777d2ccbd28" member_type: VOTER last_known_addr { host: "127.6.250.124" port: 32777 } }
00:28:15.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:15.064781 7432 sys_catalog.cc:455] T 00000000000000000000000000000000 P 305bbef030aa4dbeb2c0bcd5718f0053 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "305bbef030aa4dbeb2c0bcd5718f0053" member_type: VOTER last_known_addr { host: "127.6.250.126" port: 46847 } } peers { permanent_uuid: "791e8bcb122c4c3e98c5e01b5217df0f" member_type: VOTER last_known_addr { host: "127.6.250.125" port: 41277 } } peers { permanent_uuid: "739522ae64834cf0b7591777d2ccbd28" member_type: VOTER last_known_addr { host: "127.6.250.124" port: 32777 } } }
00:28:15.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:15.065742 7432 sys_catalog.cc:458] T 00000000000000000000000000000000 P 305bbef030aa4dbeb2c0bcd5718f0053 [sys.catalog]: This master's current role is: FOLLOWER
00:28:15.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:15.066622 7225 sys_catalog.cc:564] T 00000000000000000000000000000000 P 305bbef030aa4dbeb2c0bcd5718f0053 [sys.catalog]: configured and running, proceeding with master startup.
00:28:15.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:15.067226 7198 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 305bbef030aa4dbeb2c0bcd5718f0053 [term 0 FOLLOWER]: Advancing to term 1
00:28:15.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:15.071908 7199 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "739522ae64834cf0b7591777d2ccbd28" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "305bbef030aa4dbeb2c0bcd5718f0053"
00:28:15.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:15.073510 7200 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "739522ae64834cf0b7591777d2ccbd28" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "305bbef030aa4dbeb2c0bcd5718f0053" is_pre_election: true
00:28:15.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:15.076913 7198 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 305bbef030aa4dbeb2c0bcd5718f0053 [term 1 FOLLOWER]: Refusing update from remote peer 739522ae64834cf0b7591777d2ccbd28: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
00:28:15.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:15.078639 7417 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 739522ae64834cf0b7591777d2ccbd28 [LEADER]: Connected to new peer: Peer: permanent_uuid: "305bbef030aa4dbeb2c0bcd5718f0053" member_type: VOTER last_known_addr { host: "127.6.250.126" port: 46847 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
00:28:15.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:15.109695 7179 tablet.cc:2369] T 00000000000000000000000000000000 P 305bbef030aa4dbeb2c0bcd5718f0053: Can't schedule compaction. Clean time has not been advanced past its initial value.
00:28:15.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:15.137269 7425 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
00:28:15.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:15.137894 7425 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:28:15.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:15.138643 7425 flags.cc:425] Enabled unsafe flag: --never_fsync=true
00:28:15.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:15.177961 7432 sys_catalog.cc:455] T 00000000000000000000000000000000 P 305bbef030aa4dbeb2c0bcd5718f0053 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 739522ae64834cf0b7591777d2ccbd28. Latest consensus state: current_term: 1 leader_uuid: "739522ae64834cf0b7591777d2ccbd28" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "305bbef030aa4dbeb2c0bcd5718f0053" member_type: VOTER last_known_addr { host: "127.6.250.126" port: 46847 } } peers { permanent_uuid: "791e8bcb122c4c3e98c5e01b5217df0f" member_type: VOTER last_known_addr { host: "127.6.250.125" port: 41277 } } peers { permanent_uuid: "739522ae64834cf0b7591777d2ccbd28" member_type: VOTER last_known_addr { host: "127.6.250.124" port: 32777 } } }
00:28:15.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:15.179216 7432 sys_catalog.cc:458] T 00000000000000000000000000000000 P 305bbef030aa4dbeb2c0bcd5718f0053 [sys.catalog]: This master's current role is: FOLLOWER
00:28:15.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:15.180140 7444 sys_catalog.cc:455] T 00000000000000000000000000000000 P 305bbef030aa4dbeb2c0bcd5718f0053 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "739522ae64834cf0b7591777d2ccbd28" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "305bbef030aa4dbeb2c0bcd5718f0053" member_type: VOTER last_known_addr { host: "127.6.250.126" port: 46847 } } peers { permanent_uuid: "791e8bcb122c4c3e98c5e01b5217df0f" member_type: VOTER last_known_addr { host: "127.6.250.125" port: 41277 } } peers { permanent_uuid: "739522ae64834cf0b7591777d2ccbd28" member_type: VOTER last_known_addr { host: "127.6.250.124" port: 32777 } } }
00:28:15.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:15.181347 7444 sys_catalog.cc:458] T 00000000000000000000000000000000 P 305bbef030aa4dbeb2c0bcd5718f0053 [sys.catalog]: This master's current role is: FOLLOWER
00:28:15.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:15.195312 7449 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
00:28:15.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:15.200919 7425 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
00:28:15.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:15.202342 7425 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.6.250.65
00:28:15.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:15.258440 7415 catalog_manager.cc:1259] Loaded cluster ID: 3346e965890b4475b56752dccda948ea
00:28:15.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:15.259065 7415 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 791e8bcb122c4c3e98c5e01b5217df0f: loading cluster ID for follower catalog manager: success
00:28:15.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:15.266801 7425 tablet_server_runner.cc:78] Tablet server non-default flags:
00:28:15.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:15.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.250.84:46577
00:28:15.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:15.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:28:15.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/ts-0/data
00:28:15.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/ts-0/wal
00:28:15.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:15.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:15.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.250.65:0
00:28:15.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:28:15.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:15.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:15.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/ts-0/data/info.pb
00:28:15.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.250.65
00:28:15.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:28:15.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.250.126:46847,127.6.250.125:41277,127.6.250.124:32777
00:28:15.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:28:15.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:15.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:28:15.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:28:15.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:15.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.250.65
00:28:15.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/ts-0/logs
00:28:15.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:15.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:28:15.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
00:28:15.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:28:15.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
00:28:15.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
00:28:15.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:28:15.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 23 Sep 2026 23:32:10 UTC on ccycloud.quasar-wfxeao.root.comops.site
00:28:15.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4668917
00:28:15.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
00:28:15.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:15.268347 7425 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:28:15.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:15.270364 7425 file_cache.cc:493] Constructed file cache file cache with capacity 419430
00:28:15.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:15.280339 7415 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 791e8bcb122c4c3e98c5e01b5217df0f: acquiring CA information for follower catalog manager: success
00:28:15.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:15.286412 7415 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 791e8bcb122c4c3e98c5e01b5217df0f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
00:28:15.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:15.299414 7451 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:15.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:15.581835 7417 sys_catalog.cc:455] T 00000000000000000000000000000000 P 739522ae64834cf0b7591777d2ccbd28 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "739522ae64834cf0b7591777d2ccbd28" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "305bbef030aa4dbeb2c0bcd5718f0053" member_type: VOTER last_known_addr { host: "127.6.250.126" port: 46847 } } peers { permanent_uuid: "791e8bcb122c4c3e98c5e01b5217df0f" member_type: VOTER last_known_addr { host: "127.6.250.125" port: 41277 } } peers { permanent_uuid: "739522ae64834cf0b7591777d2ccbd28" member_type: VOTER last_known_addr { host: "127.6.250.124" port: 32777 } } }
00:28:15.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:15.582928 7417 sys_catalog.cc:458] T 00000000000000000000000000000000 P 739522ae64834cf0b7591777d2ccbd28 [sys.catalog]: This master's current role is: LEADER
00:28:16.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:16.217217 7449 catalog_manager.cc:1259] Loaded cluster ID: 3346e965890b4475b56752dccda948ea
00:28:16.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:16.217633 7449 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 305bbef030aa4dbeb2c0bcd5718f0053: loading cluster ID for follower catalog manager: success
00:28:16.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:16.241387 7449 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 305bbef030aa4dbeb2c0bcd5718f0053: acquiring CA information for follower catalog manager: success
00:28:16.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:16.249228 7449 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 305bbef030aa4dbeb2c0bcd5718f0053: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
00:28:17.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:15.299629 7452 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:17.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:16.692483 7450 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 7425
00:28:17.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:17.181614 7450 kernel_stack_watchdog.cc:198] Thread 7425 stuck at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:637 for 394ms:
00:28:17.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
00:28:17.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
00:28:17.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
00:28:17.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
00:28:17.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:17.182788 7425 thread.cc:636] GCE (cloud detector) Time spent creating pthread: real 1.886s user 0.682s sys 0.968s
00:28:17.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:17.183291 7425 thread.cc:602] GCE (cloud detector) Time spent starting thread: real 1.887s user 0.682s sys 0.968s
00:28:17.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:17.196832 7425 server_base.cc:1034] running on GCE node
00:28:17.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:17.197846 7459 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:17.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:17.199295 7425 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:28:17.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:17.201853 7425 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:28:17.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:17.204629 7425 hybrid_clock.cc:648] HybridClock initialized: now 1790209697204678 us; error 136 us; skew 500 ppm
00:28:17.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:17.207067 7425 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:28:17.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:17.236191 7425 webserver.cc:460] Webserver started at http://127.6.250.65:40243/ using document root <none> and password file <none>
00:28:17.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:17.237448 7425 fs_manager.cc:359] Metadata directory not provided
00:28:17.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:17.237720 7425 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:28:17.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:17.238246 7425 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
00:28:17.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:17.245342 7425 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/ts-0/data/instance:
00:28:17.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6a95e64aa576436ab2194a46897dc406"
00:28:17.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 00:28:17 on dist-test-slave-dnss"
00:28:17.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:17.246814 7425 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/ts-0/wal/instance:
00:28:17.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6a95e64aa576436ab2194a46897dc406"
00:28:17.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 00:28:17 on dist-test-slave-dnss"
00:28:17.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:17.261754 7425 fs_manager.cc:691] Time spent creating directory manager: real 0.014s user 0.014s sys 0.002s
00:28:17.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:17.275266 7465 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
00:28:17.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:17.276599 7425 fs_manager.cc:725] Time spent opening block manager: real 0.010s user 0.004s sys 0.004s
00:28:17.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:17.276970 7425 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/ts-0/data,/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/ts-0/wal
00:28:17.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6a95e64aa576436ab2194a46897dc406"
00:28:17.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 00:28:17 on dist-test-slave-dnss"
00:28:17.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:17.277432 7425 fs_report.cc:383] FS layout report
00:28:17.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:28:17.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/ts-0/wal
00:28:17.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/ts-0/wal
00:28:17.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/ts-0/data/data
00:28:17.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:28:17.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:28:17.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:28:17.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:28:17.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:28:17.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:28:17.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:28:17.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:28:17.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:28:17.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:28:17.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:28:17.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:17.324704 7425 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
00:28:17.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:17.326553 7425 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:28:17.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:17.327087 7425 kserver.cc:163] Server-wide thread pool size limit: 3276
00:28:17.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:17.330608 7425 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
00:28:17.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:17.336483 7425 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
00:28:17.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:17.336750 7425 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
00:28:17.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:17.337018 7425 ts_tablet_manager.cc:610] Registered 0 tablets
00:28:17.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:17.337235 7425 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
00:28:17.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:17.532510 7425 rpc_server.cc:307] RPC server started. Bound to: 127.6.250.65:39193
00:28:17.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:17.532610 7577 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.250.65:39193 every 8 connection(s)
00:28:17.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:17.535231 7425 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/ts-0/data/info.pb
00:28:17.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:17.544279 7145 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskNE0Lyu/build/tsan/bin/kudu as pid 7425
00:28:17.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:17.544719 7145 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/ts-0/wal/instance
00:28:17.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:17.572463 7145 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskNE0Lyu/build/tsan/bin/kudu
00:28:17.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNE0Lyu/build/tsan/bin/kudu
00:28:17.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/ts-1/wal
00:28:17.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/ts-1/data
00:28:17.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:28:17.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:28:17.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:28:17.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:17.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:17.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:17.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/ts-1/logs
00:28:17.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/ts-1/data/info.pb
00:28:17.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:17.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:28:17.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:28:17.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:28:17.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:28:17.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:17.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:17.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:17.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:28:17.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:28:17.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.250.66:0
00:28:17.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.250.66
00:28:17.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.250.66
00:28:17.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:28:17.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.250.126:46847,127.6.250.125:41277,127.6.250.124:32777
00:28:17.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.250.84:46577
00:28:17.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:17.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:17.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:28:17.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:17.633970 7578 heartbeater.cc:347] Connected to a master server at 127.6.250.124:32777
00:28:17.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:17.634508 7578 heartbeater.cc:464] Registering TS with master...
00:28:17.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:17.635762 7578 heartbeater.cc:511] Master 127.6.250.124:32777 requested a full tablet report, sending...
00:28:17.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:17.639523 7345 ts_manager.cc:194] Registered new tserver with Master: 6a95e64aa576436ab2194a46897dc406 (127.6.250.65:39193)
00:28:17.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:17.642360 7345 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.6.250.65:37573
00:28:17.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:17.646306 7582 heartbeater.cc:347] Connected to a master server at 127.6.250.126:46847
00:28:17.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:17.646638 7582 heartbeater.cc:464] Registering TS with master...
00:28:17.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:17.647356 7582 heartbeater.cc:511] Master 127.6.250.126:46847 requested a full tablet report, sending...
00:28:17.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:17.650635 7190 ts_manager.cc:194] Registered new tserver with Master: 6a95e64aa576436ab2194a46897dc406 (127.6.250.65:39193)
00:28:17.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:17.651726 7579 heartbeater.cc:347] Connected to a master server at 127.6.250.125:41277
00:28:17.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:17.652081 7579 heartbeater.cc:464] Registering TS with master...
00:28:17.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:17.652777 7579 heartbeater.cc:511] Master 127.6.250.125:41277 requested a full tablet report, sending...
00:28:17.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:17.655799 7265 ts_manager.cc:194] Registered new tserver with Master: 6a95e64aa576436ab2194a46897dc406 (127.6.250.65:39193)
00:28:17.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:17.977913 7585 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
00:28:17.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:17.978404 7585 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:28:17.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:17.978945 7585 flags.cc:425] Enabled unsafe flag: --never_fsync=true
00:28:18.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:18.010382 7585 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
00:28:18.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:18.011170 7585 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.6.250.66
00:28:18.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:18.050326 7585 tablet_server_runner.cc:78] Tablet server non-default flags:
00:28:18.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:18.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.250.84:46577
00:28:18.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:18.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:28:18.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/ts-1/data
00:28:18.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/ts-1/wal
00:28:18.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:18.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:18.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.250.66:0
00:28:18.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:28:18.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:18.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:18.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/ts-1/data/info.pb
00:28:18.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.250.66
00:28:18.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:28:18.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.250.126:46847,127.6.250.125:41277,127.6.250.124:32777
00:28:18.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:28:18.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:18.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:28:18.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:28:18.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:18.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.250.66
00:28:18.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/ts-1/logs
00:28:18.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:18.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:28:18.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
00:28:18.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:28:18.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
00:28:18.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
00:28:18.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:28:18.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 23 Sep 2026 23:32:10 UTC on ccycloud.quasar-wfxeao.root.comops.site
00:28:18.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4668917
00:28:18.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
00:28:18.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:18.051481 7585 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:28:18.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:18.053968 7585 file_cache.cc:493] Constructed file cache file cache with capacity 419430
00:28:18.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:18.081630 7596 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:18.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:18.649525 7578 heartbeater.cc:503] Master 127.6.250.124:32777 was elected leader, sending a full tablet report...
00:28:19.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:18.081944 7595 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:19.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:18.083593 7598 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:19.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:19.307860 7597 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
00:28:19.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:19.307964 7585 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
00:28:19.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:19.312053 7585 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:28:19.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:19.314442 7585 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:28:19.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:19.315769 7585 hybrid_clock.cc:648] HybridClock initialized: now 1790209699315757 us; error 67 us; skew 500 ppm
00:28:19.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:19.316608 7585 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:28:19.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:19.323447 7585 webserver.cc:460] Webserver started at http://127.6.250.66:43069/ using document root <none> and password file <none>
00:28:19.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:19.324509 7585 fs_manager.cc:359] Metadata directory not provided
00:28:19.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:19.324753 7585 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:28:19.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:19.325268 7585 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
00:28:19.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:19.330469 7585 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/ts-1/data/instance:
00:28:19.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fa263f1bd8ca4c1f80d9d7c721d46784"
00:28:19.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 00:28:19 on dist-test-slave-dnss"
00:28:19.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:19.331627 7585 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/ts-1/wal/instance:
00:28:19.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fa263f1bd8ca4c1f80d9d7c721d46784"
00:28:19.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 00:28:19 on dist-test-slave-dnss"
00:28:19.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:19.338719 7585 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.005s sys 0.000s
00:28:19.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:19.344372 7606 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
00:28:19.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:19.345553 7585 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
00:28:19.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:19.346026 7585 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/ts-1/data,/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/ts-1/wal
00:28:19.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fa263f1bd8ca4c1f80d9d7c721d46784"
00:28:19.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 00:28:19 on dist-test-slave-dnss"
00:28:19.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:19.346424 7585 fs_report.cc:383] FS layout report
00:28:19.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:28:19.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/ts-1/wal
00:28:19.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/ts-1/wal
00:28:19.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/ts-1/data/data
00:28:19.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:28:19.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:28:19.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:28:19.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:28:19.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:28:19.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:28:19.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:28:19.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:28:19.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:28:19.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:28:19.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:28:19.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:19.386252 7585 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
00:28:19.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:19.387881 7585 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:28:19.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:19.388425 7585 kserver.cc:163] Server-wide thread pool size limit: 3276
00:28:19.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:19.391103 7585 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
00:28:19.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:19.395426 7585 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
00:28:19.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:19.395632 7585 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
00:28:19.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:19.395942 7585 ts_tablet_manager.cc:610] Registered 0 tablets
00:28:19.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:19.396080 7585 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
00:28:19.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:19.543510 7585 rpc_server.cc:307] RPC server started. Bound to: 127.6.250.66:35735
00:28:19.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:19.543629 7718 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.250.66:35735 every 8 connection(s)
00:28:19.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:19.546114 7585 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/ts-1/data/info.pb
00:28:19.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:19.549504 7145 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskNE0Lyu/build/tsan/bin/kudu as pid 7585
00:28:19.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:19.550271 7145 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/ts-1/wal/instance
00:28:19.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:19.576778 7145 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskNE0Lyu/build/tsan/bin/kudu
00:28:19.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNE0Lyu/build/tsan/bin/kudu
00:28:19.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/ts-2/wal
00:28:19.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/ts-2/data
00:28:19.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:28:19.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:28:19.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:28:19.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:19.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:19.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:19.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/ts-2/logs
00:28:19.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/ts-2/data/info.pb
00:28:19.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:19.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:28:19.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:28:19.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:28:19.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:28:19.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:19.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:19.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:19.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
00:28:19.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:28:19.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.250.67:0
00:28:19.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.250.67
00:28:19.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.250.67
00:28:19.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:28:19.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.250.126:46847,127.6.250.125:41277,127.6.250.124:32777
00:28:19.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.250.84:46577
00:28:19.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:19.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:19.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
00:28:19.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:19.625795 7719 heartbeater.cc:347] Connected to a master server at 127.6.250.124:32777
00:28:19.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:19.626313 7719 heartbeater.cc:464] Registering TS with master...
00:28:19.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:19.628506 7719 heartbeater.cc:511] Master 127.6.250.124:32777 requested a full tablet report, sending...
00:28:19.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:19.631613 7345 ts_manager.cc:194] Registered new tserver with Master: fa263f1bd8ca4c1f80d9d7c721d46784 (127.6.250.66:35735)
00:28:19.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:19.633673 7345 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.6.250.66:58535
00:28:19.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:19.634366 7721 heartbeater.cc:347] Connected to a master server at 127.6.250.126:46847
00:28:19.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:19.634708 7721 heartbeater.cc:464] Registering TS with master...
00:28:19.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:19.635533 7721 heartbeater.cc:511] Master 127.6.250.126:46847 requested a full tablet report, sending...
00:28:19.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:19.636642 7720 heartbeater.cc:347] Connected to a master server at 127.6.250.125:41277
00:28:19.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:19.636994 7720 heartbeater.cc:464] Registering TS with master...
00:28:19.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:19.637773 7720 heartbeater.cc:511] Master 127.6.250.125:41277 requested a full tablet report, sending...
00:28:19.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:19.638301 7190 ts_manager.cc:194] Registered new tserver with Master: fa263f1bd8ca4c1f80d9d7c721d46784 (127.6.250.66:35735)
00:28:19.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:19.641160 7265 ts_manager.cc:194] Registered new tserver with Master: fa263f1bd8ca4c1f80d9d7c721d46784 (127.6.250.66:35735)
00:28:19.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:19.929495 7729 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
00:28:19.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:19.929977 7729 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:28:19.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:19.930405 7729 flags.cc:425] Enabled unsafe flag: --never_fsync=true
00:28:19.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:19.962021 7729 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
00:28:19.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:19.962818 7729 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.6.250.67
00:28:19.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:19.997047 7729 tablet_server_runner.cc:78] Tablet server non-default flags:
00:28:19.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:19.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.250.84:46577
00:28:19.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:19.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:28:19.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/ts-2/data
00:28:19.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/ts-2/wal
00:28:19.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:19.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:19.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.250.67:0
00:28:19.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:28:19.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:19.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:19.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/ts-2/data/info.pb
00:28:19.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.250.67
00:28:19.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:28:19.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.250.126:46847,127.6.250.125:41277,127.6.250.124:32777
00:28:19.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:28:19.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:19.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:28:19.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:28:19.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:19.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.250.67
00:28:19.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/ts-2/logs
00:28:19.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:19.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:28:19.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
00:28:19.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
00:28:19.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
00:28:19.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
00:28:19.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:28:19.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 23 Sep 2026 23:32:10 UTC on ccycloud.quasar-wfxeao.root.comops.site
00:28:19.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4668917
00:28:19.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
00:28:19.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:19.998172 7729 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:28:20.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:19.999773 7729 file_cache.cc:493] Constructed file cache file cache with capacity 419430
00:28:20.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:20.020675 7738 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:20.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:20.644191 7719 heartbeater.cc:503] Master 127.6.250.124:32777 was elected leader, sending a full tablet report...
00:28:21.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:20.021840 7735 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:21.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:20.022187 7729 server_base.cc:1034] running on GCE node
00:28:21.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:20.020859 7736 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:21.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:21.231339 7729 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:28:21.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:21.234162 7729 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:28:21.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:21.235610 7729 hybrid_clock.cc:648] HybridClock initialized: now 1790209701235567 us; error 60 us; skew 500 ppm
00:28:21.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:21.236506 7729 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:28:21.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:21.248301 7729 webserver.cc:460] Webserver started at http://127.6.250.67:46649/ using document root <none> and password file <none>
00:28:21.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:21.249184 7729 fs_manager.cc:359] Metadata directory not provided
00:28:21.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:21.249374 7729 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:28:21.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:21.249742 7729 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
00:28:21.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:21.254628 7729 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/ts-2/data/instance:
00:28:21.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5632462b6c2b47a28cb88201c6e67521"
00:28:21.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 00:28:21 on dist-test-slave-dnss"
00:28:21.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:21.255633 7729 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/ts-2/wal/instance:
00:28:21.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5632462b6c2b47a28cb88201c6e67521"
00:28:21.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 00:28:21 on dist-test-slave-dnss"
00:28:21.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:21.262454 7729 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.005s sys 0.000s
00:28:21.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:21.268360 7747 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
00:28:21.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:21.269519 7729 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
00:28:21.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:21.269816 7729 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/ts-2/data,/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/ts-2/wal
00:28:21.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5632462b6c2b47a28cb88201c6e67521"
00:28:21.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 00:28:21 on dist-test-slave-dnss"
00:28:21.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:21.270124 7729 fs_report.cc:383] FS layout report
00:28:21.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:28:21.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/ts-2/wal
00:28:21.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/ts-2/wal
00:28:21.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/ts-2/data/data
00:28:21.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:28:21.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:28:21.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:28:21.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:28:21.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:28:21.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:28:21.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:28:21.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:28:21.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:28:21.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:28:21.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:28:21.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:21.321237 7729 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
00:28:21.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:21.322961 7729 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:28:21.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:21.323468 7729 kserver.cc:163] Server-wide thread pool size limit: 3276
00:28:21.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:21.326428 7729 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
00:28:21.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:21.332226 7729 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
00:28:21.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:21.332454 7729 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
00:28:21.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:21.332808 7729 ts_tablet_manager.cc:610] Registered 0 tablets
00:28:21.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:21.333005 7729 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
00:28:21.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:21.475332 7729 rpc_server.cc:307] RPC server started. Bound to: 127.6.250.67:35509
00:28:21.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:21.475458 7859 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.250.67:35509 every 8 connection(s)
00:28:21.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:21.477890 7729 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/ts-2/data/info.pb
00:28:21.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:21.482266 7145 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskNE0Lyu/build/tsan/bin/kudu as pid 7729
00:28:21.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:21.482746 7145 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster4864106019962461893/ts-2/wal/instance
00:28:21.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:21.552982 7860 heartbeater.cc:347] Connected to a master server at 127.6.250.124:32777
00:28:21.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:21.553406 7860 heartbeater.cc:464] Registering TS with master...
00:28:21.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:21.554661 7860 heartbeater.cc:511] Master 127.6.250.124:32777 requested a full tablet report, sending...
00:28:21.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:21.556090 7861 heartbeater.cc:347] Connected to a master server at 127.6.250.125:41277
00:28:21.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:21.556413 7861 heartbeater.cc:464] Registering TS with master...
00:28:21.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:21.557220 7861 heartbeater.cc:511] Master 127.6.250.125:41277 requested a full tablet report, sending...
00:28:21.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:21.557682 7345 ts_manager.cc:194] Registered new tserver with Master: 5632462b6c2b47a28cb88201c6e67521 (127.6.250.67:35509)
00:28:21.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:21.559281 7345 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.6.250.67:47355
00:28:21.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:21.561545 7265 ts_manager.cc:194] Registered new tserver with Master: 5632462b6c2b47a28cb88201c6e67521 (127.6.250.67:35509)
00:28:21.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:21.567879 7864 heartbeater.cc:347] Connected to a master server at 127.6.250.126:46847
00:28:21.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:21.568178 7864 heartbeater.cc:464] Registering TS with master...
00:28:21.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:21.568943 7864 heartbeater.cc:511] Master 127.6.250.126:46847 requested a full tablet report, sending...
00:28:21.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:21.571426 7190 ts_manager.cc:194] Registered new tserver with Master: 5632462b6c2b47a28cb88201c6e67521 (127.6.250.67:35509)
00:28:21.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:21.580614 7145 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
00:28:21.592 [DEBUG - main] (MiniKuduCluster.java:195) Response:
00:28:21.593 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
00:28:21.605 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.6.250.126"
port: 46847
}
bound_http_address {
host: "127.6.250.126"
port: 39143
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.6.250.125"
port: 41277
}
bound_http_address {
host: "127.6.250.125"
port: 45901
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.6.250.124"
port: 32777
}
bound_http_address {
host: "127.6.250.124"
port: 42699
}
}
}
00:28:21.612 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
00:28:21.616 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.6.250.65"
port: 39193
}
bound_http_address {
host: "127.6.250.65"
port: 40243
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.6.250.66"
port: 35735
}
bound_http_address {
host: "127.6.250.66"
port: 43069
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.6.250.67"
port: 35509
}
bound_http_address {
host: "127.6.250.67"
port: 46649
}
}
}
00:28:21.616 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
00:28:21.801 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
00:28:21.921 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskNE0Lyu/test-tmp
00:28:21.922 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
00:28:21.922 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskNE0Lyu/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
00:28:21.927 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
00:28:22.277 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
00:28:22.278 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
00:28:22.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:22.281035 7871 mini_oidc.cc:152] Starting JWKS server
00:28:22.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:22.284636 7871 webserver.cc:460] Webserver started at http://127.0.0.1:43589/ using document root <none> and password file <none>
00:28:22.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:22.285044 7871 mini_oidc.cc:189] Starting OIDC Discovery server
00:28:22.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:22.286756 7871 webserver.cc:460] Webserver started at http://127.0.0.1:38469/ using document root <none> and password file <none>
00:28:22.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
00:28:22.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/krb5kdc/principal' for realm 'KRBTEST.COM',
00:28:22.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
00:28:22.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 00:28:22 dist-test-slave-dnss krb5kdc[7884](info): setting up network...
00:28:22.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
00:28:22.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 00:28:22 dist-test-slave-dnss krb5kdc[7884](info): set up 2 sockets
00:28:22.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 00:28:22 dist-test-slave-dnss krb5kdc[7884](info): commencing operation
00:28:22.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
00:28:22.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:22.563176 7860 heartbeater.cc:503] Master 127.6.250.124:32777 was elected leader, sending a full tablet report...
00:28:23.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:23.417913 7871 mini_kdc.cc:121] Time spent starting KDC: real 1.104s user 0.003s sys 0.017s
00:28:23.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
00:28:23.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
00:28:23.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
00:28:23.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
00:28:23.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
00:28:23.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
00:28:23.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
00:28:23.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
00:28:23.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
00:28:23.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
00:28:23.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/krb5kdc/test-user.keytab.
00:28:23.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/krb5kdc/test-user.keytab.
00:28:23.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 00:28:23 dist-test-slave-dnss krb5kdc[7884](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790209703, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
00:28:23.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
00:28:23.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-24T00:28:23Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
00:28:23.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-24T00:28:23Z Disabled control of system clock
00:28:23.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.7.175.254@KRBTEST.COM; defaulting to no policy
00:28:23.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
00:28:23.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.7.175.254@KRBTEST.COM" created.
00:28:23.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
00:28:23.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.7.175.254 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/krb5kdc/kudu_127.7.175.254.keytab.
00:28:23.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.7.175.254 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/krb5kdc/kudu_127.7.175.254.keytab.
00:28:23.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:23.629631 7871 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskNE0Lyu/build/tsan/bin/kudu
00:28:23.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNE0Lyu/build/tsan/bin/kudu
00:28:23.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-0/wal
00:28:23.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-0/data
00:28:23.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:28:23.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:28:23.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:28:23.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:23.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:23.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:23.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-0/logs
00:28:23.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-0/data/info.pb
00:28:23.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:23.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:28:23.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:28:23.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:28:23.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:28:23.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:23.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:23.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:23.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
00:28:23.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:28:23.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:28:23.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:28:23.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.175.254:39253
00:28:23.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.175.254
00:28:23.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:28:23.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/krb5kdc/kudu_127.7.175.254.keytab
00:28:23.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.175.254
00:28:23.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
00:28:23.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
00:28:23.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
00:28:23.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.175.212:41129
00:28:23.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:23.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:23.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:28:23.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:28:23.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.175.254:39253,127.7.175.253:43719,127.7.175.252:42903
00:28:23.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:28:23.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43589/jwks.json
00:28:23.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
00:28:23.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:23.940380 7897 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
00:28:23.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:23.941052 7897 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:28:23.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:23.941766 7897 flags.cc:425] Enabled unsafe flag: --never_fsync=true
00:28:23.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:23.973031 7897 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
00:28:23.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:23.973338 7897 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
00:28:23.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:23.973765 7897 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
00:28:23.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:23.974100 7897 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
00:28:23.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:23.974422 7897 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
00:28:23.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:23.974738 7897 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
00:28:23.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:23.974973 7897 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43589/jwks.json
00:28:24.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:24.010701 7897 master_runner.cc:386] Master server non-default flags:
00:28:24.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:24.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.175.212:41129
00:28:24.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:24.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:28:24.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-0/data
00:28:24.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-0/wal
00:28:24.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
00:28:24.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:28:24.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.175.254:39253,127.7.175.253:43719,127.7.175.252:42903
00:28:24.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/krb5kdc/kudu_127.7.175.254.keytab
00:28:24.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.175.254
00:28:24.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:24.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:24.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:28:24.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.175.254:39253
00:28:24.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:28:24.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:28:24.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:28:24.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43589/jwks.json
00:28:24.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
00:28:24.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
00:28:24.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
00:28:24.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:24.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:24.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-0/data/info.pb
00:28:24.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.175.254
00:28:24.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:28:24.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:28:24.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:24.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:28:24.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:28:24.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:24.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-0/logs
00:28:24.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:24.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:28:24.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
00:28:24.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
00:28:24.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
00:28:24.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
00:28:24.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:28:24.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 23 Sep 2026 23:32:10 UTC on ccycloud.quasar-wfxeao.root.comops.site
00:28:24.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4668917
00:28:24.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
00:28:24.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:24.012251 7897 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:28:24.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:24.013393 7897 file_cache.cc:493] Constructed file cache file cache with capacity 419430
00:28:24.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:24.033155 7904 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:25.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:24.033162 7903 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:25.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:24.033938 7906 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:25.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:24.035561 7897 server_base.cc:1034] running on GCE node
00:28:25.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:25.258451 7897 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:28:25.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:25.261891 7897 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:28:25.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:25.263329 7897 hybrid_clock.cc:648] HybridClock initialized: now 1790209705263287 us; error 64 us; skew 500 ppm
00:28:25.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:25.264241 7897 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:28:25.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 00:28:25 dist-test-slave-dnss krb5kdc[7884](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790209705, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.175.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
00:28:25.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:25.270507 7897 init.cc:377] Logged in from keytab as kudu/127.7.175.254@KRBTEST.COM (short username kudu)
00:28:25.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:25.275703 7897 webserver.cc:460] Webserver started at http://127.7.175.254:43191/ using document root <none> and password file <none>
00:28:25.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:25.276618 7897 fs_manager.cc:359] Metadata directory not provided
00:28:25.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:25.276806 7897 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:28:25.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:25.277175 7897 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
00:28:25.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:25.281920 7897 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-0/data/instance:
00:28:25.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aad96a77b01c43088a1359442c9eaac6"
00:28:25.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 00:28:25 on dist-test-slave-dnss"
00:28:25.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:25.282958 7897 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-0/wal/instance:
00:28:25.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aad96a77b01c43088a1359442c9eaac6"
00:28:25.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 00:28:25 on dist-test-slave-dnss"
00:28:25.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:25.289930 7897 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.006s sys 0.003s
00:28:25.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:25.295506 7915 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
00:28:25.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:25.296643 7897 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
00:28:25.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:25.296949 7897 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-0/data,/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-0/wal
00:28:25.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aad96a77b01c43088a1359442c9eaac6"
00:28:25.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 00:28:25 on dist-test-slave-dnss"
00:28:25.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:25.297294 7897 fs_report.cc:383] FS layout report
00:28:25.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:28:25.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-0/wal
00:28:25.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-0/wal
00:28:25.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-0/data/data
00:28:25.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:28:25.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:28:25.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:28:25.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:28:25.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:28:25.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:28:25.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:28:25.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:28:25.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:28:25.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:28:25.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:28:25.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:25.337985 7897 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
00:28:25.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:25.343575 7897 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:28:25.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:25.343994 7897 kserver.cc:163] Server-wide thread pool size limit: 3276
00:28:25.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:25.414207 7897 rpc_server.cc:307] RPC server started. Bound to: 127.7.175.254:39253
00:28:25.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:25.414361 7966 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.175.254:39253 every 8 connection(s)
00:28:25.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:25.417138 7897 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-0/data/info.pb
00:28:25.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:25.420396 7871 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskNE0Lyu/build/tsan/bin/kudu as pid 7897
00:28:25.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:25.420955 7871 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-0/wal/instance
00:28:25.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:25.423287 7967 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
00:28:25.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:25.437407 7967 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.175.254" port: 39253 } has no permanent_uuid. Determining permanent_uuid...
00:28:25.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.7.175.253@KRBTEST.COM; defaulting to no policy
00:28:25.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
00:28:25.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.7.175.253@KRBTEST.COM" created.
00:28:25.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 00:28:25 dist-test-slave-dnss krb5kdc[7884](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790209705, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.175.254@KRBTEST.COM for kudu/127.7.175.254@KRBTEST.COM
00:28:25.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
00:28:25.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.7.175.253 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/krb5kdc/kudu_127.7.175.253.keytab.
00:28:25.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.7.175.253 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/krb5kdc/kudu_127.7.175.253.keytab.
00:28:25.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:25.476975 7871 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskNE0Lyu/build/tsan/bin/kudu
00:28:25.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNE0Lyu/build/tsan/bin/kudu
00:28:25.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-1/wal
00:28:25.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-1/data
00:28:25.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:28:25.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:28:25.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:28:25.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:25.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:25.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:25.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-1/logs
00:28:25.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-1/data/info.pb
00:28:25.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:25.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:28:25.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:28:25.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:28:25.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:28:25.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:25.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:25.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:25.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
00:28:25.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:28:25.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:28:25.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:28:25.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.175.253:43719
00:28:25.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.175.253
00:28:25.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:28:25.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/krb5kdc/kudu_127.7.175.253.keytab
00:28:25.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.175.253
00:28:25.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
00:28:25.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
00:28:25.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
00:28:25.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.175.212:41129
00:28:25.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:25.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:25.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:28:25.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:28:25.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.175.254:39253,127.7.175.253:43719,127.7.175.252:42903
00:28:25.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:28:25.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43589/jwks.json
00:28:25.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
00:28:25.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:25.482458 7967 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.175.253" port: 43719 } has no permanent_uuid. Determining permanent_uuid...
00:28:25.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:25.484961 7919 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.175.253:43719: connect: Connection refused (error 111)
00:28:25.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:25.490439 7967 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.175.253:43719: Network error: Client connection negotiation failed: client connection to 127.7.175.253:43719: connect: Connection refused (error 111)
00:28:25.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:25.539940 7967 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.175.253" port: 43719 } attempt: 1
00:28:25.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:25.543933 7967 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.175.253:43719: Network error: Client connection negotiation failed: client connection to 127.7.175.253:43719: connect: Connection refused (error 111)
00:28:25.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:25.612350 7967 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.175.253" port: 43719 } attempt: 2
00:28:25.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:25.616050 7967 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.175.253:43719: Network error: Client connection negotiation failed: client connection to 127.7.175.253:43719: connect: Connection refused (error 111)
00:28:25.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:25.707435 7967 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.175.253" port: 43719 } attempt: 3
00:28:25.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:25.710956 7967 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.175.253:43719: Network error: Client connection negotiation failed: client connection to 127.7.175.253:43719: connect: Connection refused (error 111)
00:28:25.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:25.796248 7973 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
00:28:25.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:25.796793 7973 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:28:25.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:25.797358 7973 flags.cc:425] Enabled unsafe flag: --never_fsync=true
00:28:25.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:25.829727 7973 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
00:28:25.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:25.830013 7973 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
00:28:25.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:25.830281 7973 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
00:28:25.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:25.830472 7973 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
00:28:25.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:25.830668 7973 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
00:28:25.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:25.830871 7973 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
00:28:25.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:25.831013 7973 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43589/jwks.json
00:28:25.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:25.854312 7967 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.175.253" port: 43719 } attempt: 4
00:28:25.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:25.857980 7967 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.175.253:43719: Network error: Client connection negotiation failed: client connection to 127.7.175.253:43719: connect: Connection refused (error 111)
00:28:25.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:25.869587 7973 master_runner.cc:386] Master server non-default flags:
00:28:25.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:25.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.175.212:41129
00:28:25.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:25.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:28:25.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-1/data
00:28:25.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-1/wal
00:28:25.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
00:28:25.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:28:25.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.175.254:39253,127.7.175.253:43719,127.7.175.252:42903
00:28:25.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/krb5kdc/kudu_127.7.175.253.keytab
00:28:25.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.175.253
00:28:25.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:25.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:25.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:28:25.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.175.253:43719
00:28:25.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:28:25.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:28:25.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:28:25.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43589/jwks.json
00:28:25.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
00:28:25.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
00:28:25.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
00:28:25.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:25.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:25.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-1/data/info.pb
00:28:25.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.175.253
00:28:25.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:28:25.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:28:25.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:25.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:28:25.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:28:25.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:25.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-1/logs
00:28:25.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:25.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:28:25.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
00:28:25.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
00:28:25.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
00:28:25.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
00:28:25.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:28:25.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 23 Sep 2026 23:32:10 UTC on ccycloud.quasar-wfxeao.root.comops.site
00:28:25.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4668917
00:28:25.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
00:28:25.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:25.870981 7973 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:28:25.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:25.872330 7973 file_cache.cc:493] Constructed file cache file cache with capacity 419430
00:28:25.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:25.888868 7983 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:26.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:26.157361 7967 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.175.253" port: 43719 } attempt: 5
00:28:26.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:26.162307 7967 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.175.253:43719: Network error: Client connection negotiation failed: client connection to 127.7.175.253:43719: connect: Connection refused (error 111)
00:28:26.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:26.709738 7967 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.175.253" port: 43719 } attempt: 6
00:28:26.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:26.717823 7967 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.175.253:43719: Network error: Client connection negotiation failed: client connection to 127.7.175.253:43719: connect: Connection refused (error 111)
00:28:27.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:25.888896 7980 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:27.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:25.890882 7973 server_base.cc:1034] running on GCE node
00:28:27.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:25.889209 7981 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:27.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:27.125069 7973 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:28:27.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:27.127557 7973 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:28:27.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:27.129083 7973 hybrid_clock.cc:648] HybridClock initialized: now 1790209707129055 us; error 49 us; skew 500 ppm
00:28:27.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:27.129953 7973 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:28:27.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 00:28:27 dist-test-slave-dnss krb5kdc[7884](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790209707, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.175.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
00:28:27.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:27.135468 7973 init.cc:377] Logged in from keytab as kudu/127.7.175.253@KRBTEST.COM (short username kudu)
00:28:27.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:27.140856 7973 webserver.cc:460] Webserver started at http://127.7.175.253:46185/ using document root <none> and password file <none>
00:28:27.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:27.141716 7973 fs_manager.cc:359] Metadata directory not provided
00:28:27.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:27.141896 7973 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:28:27.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:27.142278 7973 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
00:28:27.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:27.147393 7973 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-1/data/instance:
00:28:27.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a6b49e0918be4012bcf4db0751e4496f"
00:28:27.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 00:28:27 on dist-test-slave-dnss"
00:28:27.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:27.148526 7973 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-1/wal/instance:
00:28:27.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a6b49e0918be4012bcf4db0751e4496f"
00:28:27.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 00:28:27 on dist-test-slave-dnss"
00:28:27.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:27.156261 7973 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
00:28:27.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:27.161746 7993 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
00:28:27.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:27.163024 7973 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.001s
00:28:27.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:27.163321 7973 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-1/data,/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-1/wal
00:28:27.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a6b49e0918be4012bcf4db0751e4496f"
00:28:27.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 00:28:27 on dist-test-slave-dnss"
00:28:27.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:27.163605 7973 fs_report.cc:383] FS layout report
00:28:27.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:28:27.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-1/wal
00:28:27.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-1/wal
00:28:27.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-1/data/data
00:28:27.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:28:27.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:28:27.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:28:27.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:28:27.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:28:27.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:28:27.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:28:27.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:28:27.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:28:27.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:28:27.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:28:27.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:27.205907 7973 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
00:28:27.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:27.211090 7973 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:28:27.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:27.211509 7973 kserver.cc:163] Server-wide thread pool size limit: 3276
00:28:27.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:27.281374 7973 rpc_server.cc:307] RPC server started. Bound to: 127.7.175.253:43719
00:28:27.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:27.281582 8044 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.175.253:43719 every 8 connection(s)
00:28:27.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:27.284013 7973 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-1/data/info.pb
00:28:27.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:27.289386 8045 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
00:28:27.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:27.293164 7871 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskNE0Lyu/build/tsan/bin/kudu as pid 7973
00:28:27.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:27.293597 7871 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-1/wal/instance
00:28:27.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:27.302946 8045 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.175.254" port: 39253 } has no permanent_uuid. Determining permanent_uuid...
00:28:27.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.7.175.252@KRBTEST.COM; defaulting to no policy
00:28:27.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
00:28:27.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.7.175.252@KRBTEST.COM" created.
00:28:27.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 00:28:27 dist-test-slave-dnss krb5kdc[7884](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790209707, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.175.253@KRBTEST.COM for kudu/127.7.175.254@KRBTEST.COM
00:28:27.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:27.342087 8045 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.175.253" port: 43719 } has no permanent_uuid. Determining permanent_uuid...
00:28:27.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
00:28:27.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.7.175.252 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/krb5kdc/kudu_127.7.175.252.keytab.
00:28:27.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.7.175.252 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/krb5kdc/kudu_127.7.175.252.keytab.
00:28:27.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:27.345309 7871 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskNE0Lyu/build/tsan/bin/kudu
00:28:27.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNE0Lyu/build/tsan/bin/kudu
00:28:27.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-2/wal
00:28:27.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-2/data
00:28:27.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:28:27.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:28:27.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:28:27.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:27.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:27.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:27.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-2/logs
00:28:27.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-2/data/info.pb
00:28:27.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:27.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:28:27.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:28:27.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:28:27.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:28:27.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:27.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:27.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:27.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
00:28:27.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:28:27.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:28:27.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:28:27.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.175.252:42903
00:28:27.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.175.252
00:28:27.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:28:27.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/krb5kdc/kudu_127.7.175.252.keytab
00:28:27.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.175.252
00:28:27.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
00:28:27.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
00:28:27.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
00:28:27.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.175.212:41129
00:28:27.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:27.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:27.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:28:27.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:28:27.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.175.254:39253,127.7.175.253:43719,127.7.175.252:42903
00:28:27.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:28:27.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43589/jwks.json
00:28:27.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
00:28:27.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 00:28:27 dist-test-slave-dnss krb5kdc[7884](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790209707, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.175.253@KRBTEST.COM for kudu/127.7.175.253@KRBTEST.COM
00:28:27.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:27.375459 8045 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.175.252" port: 42903 } has no permanent_uuid. Determining permanent_uuid...
00:28:27.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:27.377844 7995 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.175.252:42903: connect: Connection refused (error 111)
00:28:27.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:27.381652 8045 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.175.252:42903: Network error: Client connection negotiation failed: client connection to 127.7.175.252:42903: connect: Connection refused (error 111)
00:28:27.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:27.431191 8045 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.175.252" port: 42903 } attempt: 1
00:28:27.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:27.435482 8045 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.175.252:42903: Network error: Client connection negotiation failed: client connection to 127.7.175.252:42903: connect: Connection refused (error 111)
00:28:27.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:27.503964 8045 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.175.252" port: 42903 } attempt: 2
00:28:27.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:27.507736 8045 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.175.252:42903: Network error: Client connection negotiation failed: client connection to 127.7.175.252:42903: connect: Connection refused (error 111)
00:28:27.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:27.599141 8045 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.175.252" port: 42903 } attempt: 3
00:28:27.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:27.602795 8045 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.175.252:42903: Network error: Client connection negotiation failed: client connection to 127.7.175.252:42903: connect: Connection refused (error 111)
00:28:27.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:27.685313 8052 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
00:28:27.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:27.685848 8052 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:28:27.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:27.686429 8052 flags.cc:425] Enabled unsafe flag: --never_fsync=true
00:28:27.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:27.719013 8052 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
00:28:27.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:27.719249 8052 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
00:28:27.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:27.719503 8052 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
00:28:27.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:27.719764 8052 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
00:28:27.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:27.719966 8052 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
00:28:27.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:27.720147 8052 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
00:28:27.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:27.720293 8052 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43589/jwks.json
00:28:27.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:27.746165 8045 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.175.252" port: 42903 } attempt: 4
00:28:27.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:27.749663 8045 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.175.252:42903: Network error: Client connection negotiation failed: client connection to 127.7.175.252:42903: connect: Connection refused (error 111)
00:28:27.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:27.758448 8052 master_runner.cc:386] Master server non-default flags:
00:28:27.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:27.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.175.212:41129
00:28:27.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:27.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:28:27.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-2/data
00:28:27.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-2/wal
00:28:27.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
00:28:27.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:28:27.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.175.254:39253,127.7.175.253:43719,127.7.175.252:42903
00:28:27.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/krb5kdc/kudu_127.7.175.252.keytab
00:28:27.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.175.252
00:28:27.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:27.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:27.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:28:27.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.175.252:42903
00:28:27.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:28:27.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:28:27.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:28:27.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43589/jwks.json
00:28:27.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
00:28:27.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
00:28:27.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
00:28:27.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:27.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:27.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-2/data/info.pb
00:28:27.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.175.252
00:28:27.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
00:28:27.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:28:27.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:27.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:28:27.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:28:27.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:27.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-2/logs
00:28:27.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:27.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:28:27.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
00:28:27.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
00:28:27.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
00:28:27.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
00:28:27.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:28:27.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 23 Sep 2026 23:32:10 UTC on ccycloud.quasar-wfxeao.root.comops.site
00:28:27.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4668917
00:28:27.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
00:28:27.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:27.759547 8052 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:28:27.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:27.760552 8052 file_cache.cc:493] Constructed file cache file cache with capacity 419430
00:28:27.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:27.778901 8059 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:27.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:27.783772 7967 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.175.253" port: 43719 } attempt: 7
00:28:27.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 00:28:27 dist-test-slave-dnss krb5kdc[7884](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790209705, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.175.254@KRBTEST.COM for kudu/127.7.175.253@KRBTEST.COM
00:28:27.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:27.812978 7967 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.175.252" port: 42903 } has no permanent_uuid. Determining permanent_uuid...
00:28:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:27.818637 7967 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.175.252:42903: Network error: Client connection negotiation failed: client connection to 127.7.175.252:42903: connect: Connection refused (error 111)
00:28:27.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:27.876997 7967 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.175.252" port: 42903 } attempt: 1
00:28:27.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:27.881539 7967 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.175.252:42903: Network error: Client connection negotiation failed: client connection to 127.7.175.252:42903: connect: Connection refused (error 111)
00:28:27.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:27.963014 7967 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.175.252" port: 42903 } attempt: 2
00:28:27.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:27.966677 7967 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.175.252:42903: Network error: Client connection negotiation failed: client connection to 127.7.175.252:42903: connect: Connection refused (error 111)
00:28:28.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:28.049114 8045 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.175.252" port: 42903 } attempt: 5
00:28:28.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:28.052106 7967 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.175.252" port: 42903 } attempt: 3
00:28:28.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:28.053814 8045 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.175.252:42903: Network error: Client connection negotiation failed: client connection to 127.7.175.252:42903: connect: Connection refused (error 111)
00:28:28.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:28.057828 7967 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.175.252:42903: Network error: Client connection negotiation failed: client connection to 127.7.175.252:42903: connect: Connection refused (error 111)
00:28:28.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:28.198247 7967 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.175.252" port: 42903 } attempt: 4
00:28:28.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:28.202713 7967 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.175.252:42903: Network error: Client connection negotiation failed: client connection to 127.7.175.252:42903: connect: Connection refused (error 111)
00:28:28.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:28.486168 7967 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.175.252" port: 42903 } attempt: 5
00:28:28.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:28.490808 7967 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.175.252:42903: Network error: Client connection negotiation failed: client connection to 127.7.175.252:42903: connect: Connection refused (error 111)
00:28:28.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:28.601282 8045 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.175.252" port: 42903 } attempt: 6
00:28:28.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:28.608824 8045 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.175.252:42903: Network error: Client connection negotiation failed: client connection to 127.7.175.252:42903: connect: Connection refused (error 111)
00:28:29.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:27.779412 8060 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:29.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:27.784200 8062 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:29.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:29.037798 8061 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
00:28:29.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:29.037849 8052 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
00:28:29.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:29.042001 8052 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:28:29.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:29.043255 7967 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.175.252" port: 42903 } attempt: 6
00:28:29.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:29.045001 8052 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:28:29.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:29.046752 8052 hybrid_clock.cc:648] HybridClock initialized: now 1790209709046663 us; error 84 us; skew 500 ppm
00:28:29.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:29.047910 8052 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:28:29.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:29.049129 7967 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.175.252:42903: Network error: Client connection negotiation failed: client connection to 127.7.175.252:42903: connect: Connection refused (error 111)
00:28:29.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 00:28:29 dist-test-slave-dnss krb5kdc[7884](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790209709, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.175.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
00:28:29.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:29.054376 8052 init.cc:377] Logged in from keytab as kudu/127.7.175.252@KRBTEST.COM (short username kudu)
00:28:29.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:29.059791 8052 webserver.cc:460] Webserver started at http://127.7.175.252:44139/ using document root <none> and password file <none>
00:28:29.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:29.060729 8052 fs_manager.cc:359] Metadata directory not provided
00:28:29.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:29.060957 8052 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:28:29.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:29.061455 8052 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
00:28:29.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:29.066218 8052 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-2/data/instance:
00:28:29.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4b3b2ea20cfb450d8ad78bdee67408e2"
00:28:29.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 00:28:29 on dist-test-slave-dnss"
00:28:29.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:29.067514 8052 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-2/wal/instance:
00:28:29.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4b3b2ea20cfb450d8ad78bdee67408e2"
00:28:29.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 00:28:29 on dist-test-slave-dnss"
00:28:29.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:29.074924 8052 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.000s
00:28:29.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:29.080261 8079 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
00:28:29.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:29.081411 8052 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
00:28:29.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:29.081755 8052 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-2/data,/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-2/wal
00:28:29.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4b3b2ea20cfb450d8ad78bdee67408e2"
00:28:29.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 00:28:29 on dist-test-slave-dnss"
00:28:29.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:29.082050 8052 fs_report.cc:383] FS layout report
00:28:29.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:28:29.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-2/wal
00:28:29.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-2/wal
00:28:29.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-2/data/data
00:28:29.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:28:29.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:28:29.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:28:29.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:28:29.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:28:29.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:28:29.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:28:29.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:28:29.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:28:29.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:28:29.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:28:29.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:29.117738 8052 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
00:28:29.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:29.122978 8052 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:28:29.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:29.123377 8052 kserver.cc:163] Server-wide thread pool size limit: 3276
00:28:29.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:29.193140 8052 rpc_server.cc:307] RPC server started. Bound to: 127.7.175.252:42903
00:28:29.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:29.193207 8130 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.175.252:42903 every 8 connection(s)
00:28:29.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:29.196785 8052 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-2/data/info.pb
00:28:29.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:29.199893 7871 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskNE0Lyu/build/tsan/bin/kudu as pid 8052
00:28:29.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:29.200403 7871 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-2/wal/instance
00:28:29.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:29.203105 8131 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
00:28:29.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:29.216864 8131 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.175.254" port: 39253 } has no permanent_uuid. Determining permanent_uuid...
00:28:29.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 00:28:29 dist-test-slave-dnss krb5kdc[7884](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790209703, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.7.175.252@KRBTEST.COM
00:28:29.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 00:28:29 dist-test-slave-dnss krb5kdc[7884](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790209709, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.175.252@KRBTEST.COM for kudu/127.7.175.254@KRBTEST.COM
00:28:29.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:29.251119 8131 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.175.253" port: 43719 } has no permanent_uuid. Determining permanent_uuid...
00:28:29.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 00:28:29 dist-test-slave-dnss krb5kdc[7884](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790209709, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.175.252@KRBTEST.COM for kudu/127.7.175.253@KRBTEST.COM
00:28:29.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 00:28:29 dist-test-slave-dnss krb5kdc[7884](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790209703, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.7.175.253@KRBTEST.COM
00:28:29.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:29.277402 8131 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.175.252" port: 42903 } has no permanent_uuid. Determining permanent_uuid...
00:28:29.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 00:28:29 dist-test-slave-dnss krb5kdc[7884](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790209709, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.175.252@KRBTEST.COM for kudu/127.7.175.252@KRBTEST.COM
00:28:29.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 00:28:29 dist-test-slave-dnss krb5kdc[7884](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790209703, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.7.175.254@KRBTEST.COM
00:28:29.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:29.307245 8131 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2: Bootstrap starting.
00:28:29.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:29.313854 8131 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2: Neither blocks nor log segments found. Creating new log.
00:28:29.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:29.316190 8131 log.cc:824] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2: Log is configured to *not* fsync() on all Append() calls
00:28:29.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:29.319344 7871 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
00:28:29.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:29.321157 8131 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2: No bootstrap required, opened a new log
00:28:29.326 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
00:28:29.328 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
00:28:29.341 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
00:28:29.341 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
00:28:29.345 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.7.175.254"
port: 39253
}
bound_http_address {
host: "127.7.175.254"
port: 43191
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.7.175.253"
port: 43719
}
bound_http_address {
host: "127.7.175.253"
port: 46185
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.7.175.252"
port: 42903
}
bound_http_address {
host: "127.7.175.252"
port: 44139
}
}
}
00:28:29.346 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
00:28:29.347 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
00:28:29.347 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
00:28:29.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:29.347319 8131 raft_consensus.cc:348] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aad96a77b01c43088a1359442c9eaac6" member_type: VOTER last_known_addr { host: "127.7.175.254" port: 39253 } } peers { permanent_uuid: "a6b49e0918be4012bcf4db0751e4496f" member_type: VOTER last_known_addr { host: "127.7.175.253" port: 43719 } } peers { permanent_uuid: "4b3b2ea20cfb450d8ad78bdee67408e2" member_type: VOTER last_known_addr { host: "127.7.175.252" port: 42903 } }
00:28:29.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:29.348186 8131 raft_consensus.cc:374] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
00:28:29.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:29.348479 8131 raft_consensus.cc:729] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4b3b2ea20cfb450d8ad78bdee67408e2, State: Initialized, Role: FOLLOWER
00:28:29.348 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
00:28:29.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:29.349330 8131 consensus_queue.cc:260] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aad96a77b01c43088a1359442c9eaac6" member_type: VOTER last_known_addr { host: "127.7.175.254" port: 39253 } } peers { permanent_uuid: "a6b49e0918be4012bcf4db0751e4496f" member_type: VOTER last_known_addr { host: "127.7.175.253" port: 43719 } } peers { permanent_uuid: "4b3b2ea20cfb450d8ad78bdee67408e2" member_type: VOTER last_known_addr { host: "127.7.175.252" port: 42903 } }
00:28:29.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:29.351827 8140 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aad96a77b01c43088a1359442c9eaac6" member_type: VOTER last_known_addr { host: "127.7.175.254" port: 39253 } } peers { permanent_uuid: "a6b49e0918be4012bcf4db0751e4496f" member_type: VOTER last_known_addr { host: "127.7.175.253" port: 43719 } } peers { permanent_uuid: "4b3b2ea20cfb450d8ad78bdee67408e2" member_type: VOTER last_known_addr { host: "127.7.175.252" port: 42903 } } }
00:28:29.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:29.352743 8140 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [sys.catalog]: This master's current role is: FOLLOWER
00:28:29.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:29.353467 8131 sys_catalog.cc:564] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [sys.catalog]: configured and running, proceeding with master startup.
00:28:29.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 00:28:29 dist-test-slave-dnss krb5kdc[7884](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790209709, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
00:28:29.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:29.378381 8085 tablet.cc:2369] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2: Can't schedule compaction. Clean time has not been advanced past its initial value.
00:28:29.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
00:28:29.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:29.384002 8153 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:28:29.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:29.384341 8153 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:28:29.387 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
00:28:29.390 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/krb5kdc/krb5cc
00:28:29.397 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
00:28:29.399 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/krb5kdc/krb5cc
00:28:29.401 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
00:28:29.404 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
00:28:29.407 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
00:28:29.407 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
00:28:29.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:29.380054 8052 master_runner.cc:186] Error getting master registration for 127.7.175.254:39253: OK, instance_id { permanent_uuid: "aad96a77b01c43088a1359442c9eaac6" instance_seqno: 1790209705395034 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
00:28:29.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:29.409629 8052 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
00:28:29.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 00:28:29 dist-test-slave-dnss krb5kdc[7884](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790209709, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
00:28:29.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
00:28:29.441 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
00:28:29.441 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/krb5kdc/krb5cc
00:28:29.443 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
00:28:29.443 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.7.175.254:39253
00:28:29.445 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
00:28:29.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:29.446374 7871 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskNE0Lyu/build/tsan/bin/kudu with pid 7897
00:28:29.469 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
00:28:29.469 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.7.175.252:42903
00:28:29.470 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
00:28:29.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:29.470602 7871 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskNE0Lyu/build/tsan/bin/kudu with pid 8052
00:28:29.492 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
00:28:29.492 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.7.175.253:43719
00:28:29.493 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
00:28:29.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:29.493312 7871 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskNE0Lyu/build/tsan/bin/kudu with pid 7973
00:28:29.515 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
00:28:29.515 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.7.175.254:39253
00:28:29.517 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
00:28:29.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:29.518762 7871 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskNE0Lyu/build/tsan/bin/kudu
00:28:29.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNE0Lyu/build/tsan/bin/kudu
00:28:29.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-0/wal
00:28:29.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-0/data
00:28:29.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:28:29.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:28:29.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:28:29.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:29.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:29.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:29.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-0/logs
00:28:29.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-0/data/info.pb
00:28:29.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:29.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:28:29.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:28:29.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:28:29.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:28:29.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:29.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:29.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:29.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
00:28:29.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:28:29.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:28:29.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:28:29.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.175.254:39253
00:28:29.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.175.254
00:28:29.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43191
00:28:29.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/krb5kdc/kudu_127.7.175.254.keytab
00:28:29.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.175.254
00:28:29.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
00:28:29.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
00:28:29.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
00:28:29.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.175.212:41129
00:28:29.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:29.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:29.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:28:29.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:28:29.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.175.254:39253,127.7.175.253:43719,127.7.175.252:42903
00:28:29.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:28:29.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43589/jwks.json
00:28:29.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
00:28:29.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:29.832376 8157 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
00:28:29.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:29.833099 8157 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:28:29.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:29.833890 8157 flags.cc:425] Enabled unsafe flag: --never_fsync=true
00:28:29.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:29.866014 8157 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
00:28:29.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:29.866302 8157 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
00:28:29.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:29.866700 8157 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
00:28:29.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:29.867027 8157 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
00:28:29.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:29.867328 8157 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
00:28:29.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:29.867715 8157 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
00:28:29.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:29.867956 8157 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43589/jwks.json
00:28:29.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:29.908200 8157 master_runner.cc:386] Master server non-default flags:
00:28:29.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:29.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.175.212:41129
00:28:29.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:29.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:28:29.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-0/data
00:28:29.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-0/wal
00:28:29.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
00:28:29.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:28:29.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.175.254:39253,127.7.175.253:43719,127.7.175.252:42903
00:28:29.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/krb5kdc/kudu_127.7.175.254.keytab
00:28:29.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.175.254
00:28:29.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:29.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:29.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:28:29.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.175.254:39253
00:28:29.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:28:29.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:28:29.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:28:29.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43589/jwks.json
00:28:29.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
00:28:29.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
00:28:29.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
00:28:29.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:29.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:29.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-0/data/info.pb
00:28:29.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.175.254
00:28:29.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43191
00:28:29.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:28:29.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:29.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:28:29.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:28:29.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:29.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-0/logs
00:28:29.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:29.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:28:29.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
00:28:29.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
00:28:29.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
00:28:29.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
00:28:29.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:28:29.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 23 Sep 2026 23:32:10 UTC on ccycloud.quasar-wfxeao.root.comops.site
00:28:29.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4668917
00:28:29.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
00:28:29.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:29.909684 8157 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:28:29.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:29.910893 8157 file_cache.cc:493] Constructed file cache file cache with capacity 419430
00:28:29.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:29.929592 8165 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:31.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:31.147711 7715 debug-util.cc:398] Leaking SignalData structure 0x7b08000bbce0 after lost signal to thread 7589
00:28:31.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:31.149204 7715 debug-util.cc:398] Leaking SignalData structure 0x7b08000bb540 after lost signal to thread 7718
00:28:31.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:29.931496 8164 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:31.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:29.930153 8167 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:31.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:31.238124 8166 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1305 milliseconds
00:28:31.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:31.238256 8157 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
00:28:31.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:31.239648 8157 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:28:31.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:31.242959 8157 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:28:31.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:31.244489 8157 hybrid_clock.cc:648] HybridClock initialized: now 1790209711244434 us; error 74 us; skew 500 ppm
00:28:31.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:31.245504 8157 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:28:31.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 00:28:31 dist-test-slave-dnss krb5kdc[7884](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790209711, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.175.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
00:28:31.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:31.251159 8157 init.cc:377] Logged in from keytab as kudu/127.7.175.254@KRBTEST.COM (short username kudu)
00:28:31.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:31.256915 8157 webserver.cc:460] Webserver started at http://127.7.175.254:43191/ using document root <none> and password file <none>
00:28:31.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:31.257757 8157 fs_manager.cc:359] Metadata directory not provided
00:28:31.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:31.257949 8157 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:28:31.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:31.267323 8157 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.006s sys 0.001s
00:28:31.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:31.272827 8176 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
00:28:31.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:31.273959 8157 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.000s sys 0.004s
00:28:31.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:31.274281 8157 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-0/data,/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-0/wal
00:28:31.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aad96a77b01c43088a1359442c9eaac6"
00:28:31.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 00:28:25 on dist-test-slave-dnss"
00:28:31.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:31.276651 8157 fs_report.cc:383] FS layout report
00:28:31.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:28:31.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-0/wal
00:28:31.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-0/wal
00:28:31.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-0/data/data
00:28:31.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:28:31.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:28:31.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:28:31.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:28:31.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:28:31.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:28:31.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:28:31.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:28:31.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:28:31.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:28:31.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:28:31.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:31.339046 8157 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
00:28:31.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:31.349458 8157 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:28:31.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:31.350062 8157 kserver.cc:163] Server-wide thread pool size limit: 3276
00:28:31.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:31.451954 8157 rpc_server.cc:307] RPC server started. Bound to: 127.7.175.254:39253
00:28:31.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:31.452332 8227 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.175.254:39253 every 8 connection(s)
00:28:31.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:31.455001 8157 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-0/data/info.pb
00:28:31.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:31.457695 7871 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskNE0Lyu/build/tsan/bin/kudu as pid 8157
00:28:31.458 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
00:28:31.458 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.7.175.252:42903
00:28:31.459 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
00:28:31.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:31.460675 7871 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskNE0Lyu/build/tsan/bin/kudu
00:28:31.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNE0Lyu/build/tsan/bin/kudu
00:28:31.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-2/wal
00:28:31.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-2/data
00:28:31.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:28:31.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:28:31.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:28:31.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:31.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:31.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:31.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-2/logs
00:28:31.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-2/data/info.pb
00:28:31.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:31.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:28:31.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:28:31.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:28:31.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:28:31.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:31.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:31.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:31.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
00:28:31.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:28:31.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:28:31.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:28:31.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.175.252:42903
00:28:31.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.175.252
00:28:31.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44139
00:28:31.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/krb5kdc/kudu_127.7.175.252.keytab
00:28:31.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.175.252
00:28:31.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
00:28:31.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
00:28:31.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
00:28:31.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.175.212:41129
00:28:31.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:31.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:31.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:28:31.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:28:31.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.175.254:39253,127.7.175.253:43719,127.7.175.252:42903
00:28:31.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:28:31.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43589/jwks.json
00:28:31.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
00:28:31.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:31.478437 8228 sys_catalog.cc:263] Verifying existing consensus state
00:28:31.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260924 00:28:31.479676 8228 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
00:28:31.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:31.480353 8157 master.cc:537] Master@127.7.175.254:39253 shutting down...
00:28:31.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:31.569653 8157 master.cc:552] Master@127.7.175.254:39253 shutdown complete.
00:28:31.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
00:28:31.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:31.829303 8230 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
00:28:31.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:31.829972 8230 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:28:31.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:31.830672 8230 flags.cc:425] Enabled unsafe flag: --never_fsync=true
00:28:31.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:31.862087 8230 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
00:28:31.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:31.862353 8230 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
00:28:31.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:31.862756 8230 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
00:28:31.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:31.863073 8230 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
00:28:31.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:31.863379 8230 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
00:28:31.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:31.863722 8230 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
00:28:31.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:31.863979 8230 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43589/jwks.json
00:28:31.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:31.901503 8230 master_runner.cc:386] Master server non-default flags:
00:28:31.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:31.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.175.212:41129
00:28:31.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:31.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:28:31.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-2/data
00:28:31.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-2/wal
00:28:31.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
00:28:31.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:28:31.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.175.254:39253,127.7.175.253:43719,127.7.175.252:42903
00:28:31.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/krb5kdc/kudu_127.7.175.252.keytab
00:28:31.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.175.252
00:28:31.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:31.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:31.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:28:31.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.175.252:42903
00:28:31.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:28:31.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:28:31.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:28:31.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43589/jwks.json
00:28:31.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
00:28:31.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
00:28:31.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
00:28:31.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:31.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:31.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-2/data/info.pb
00:28:31.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.175.252
00:28:31.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44139
00:28:31.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:28:31.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:31.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:28:31.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:28:31.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:31.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-2/logs
00:28:31.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:31.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:28:31.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
00:28:31.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
00:28:31.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
00:28:31.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
00:28:31.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:28:31.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 23 Sep 2026 23:32:10 UTC on ccycloud.quasar-wfxeao.root.comops.site
00:28:31.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4668917
00:28:31.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
00:28:31.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:31.902952 8230 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:28:31.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:31.904294 8230 file_cache.cc:493] Constructed file cache file cache with capacity 419430
00:28:31.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:31.924983 8237 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:33.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:31.928309 8236 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:33.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:31.927649 8239 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:33.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:33.545315 8238 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1618 milliseconds
00:28:33.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:33.545459 8230 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
00:28:33.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:33.547247 8230 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:28:33.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:33.549768 8230 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:28:33.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:33.551151 8230 hybrid_clock.cc:648] HybridClock initialized: now 1790209713551101 us; error 49 us; skew 500 ppm
00:28:33.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:33.552175 8230 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:28:33.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 00:28:33 dist-test-slave-dnss krb5kdc[7884](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790209713, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.175.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
00:28:33.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:33.565613 8230 init.cc:377] Logged in from keytab as kudu/127.7.175.252@KRBTEST.COM (short username kudu)
00:28:33.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:33.571286 8230 webserver.cc:460] Webserver started at http://127.7.175.252:44139/ using document root <none> and password file <none>
00:28:33.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:33.572333 8230 fs_manager.cc:359] Metadata directory not provided
00:28:33.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:33.572620 8230 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:28:33.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:33.582136 8230 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.005s sys 0.000s
00:28:33.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:33.586284 8249 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
00:28:33.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:33.587280 8230 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
00:28:33.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:33.587621 8230 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-2/data,/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-2/wal
00:28:33.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4b3b2ea20cfb450d8ad78bdee67408e2"
00:28:33.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 00:28:29 on dist-test-slave-dnss"
00:28:33.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:33.590018 8230 fs_report.cc:383] FS layout report
00:28:33.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:28:33.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-2/wal
00:28:33.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-2/wal
00:28:33.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-2/data/data
00:28:33.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:28:33.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:28:33.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:28:33.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:28:33.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:28:33.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:28:33.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:28:33.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:28:33.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:28:33.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:28:33.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:28:33.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:33.629091 8230 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
00:28:33.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:33.635110 8230 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:28:33.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:33.635694 8230 kserver.cc:163] Server-wide thread pool size limit: 3276
00:28:33.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:33.708330 8230 rpc_server.cc:307] RPC server started. Bound to: 127.7.175.252:42903
00:28:33.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:33.708537 8300 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.175.252:42903 every 8 connection(s)
00:28:33.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:33.711549 8230 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-2/data/info.pb
00:28:33.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:33.714123 7871 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskNE0Lyu/build/tsan/bin/kudu as pid 8230
00:28:33.715 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
00:28:33.715 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.7.175.253:43719
00:28:33.716 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
00:28:33.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:33.717409 7871 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskNE0Lyu/build/tsan/bin/kudu
00:28:33.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNE0Lyu/build/tsan/bin/kudu
00:28:33.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-1/wal
00:28:33.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-1/data
00:28:33.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
00:28:33.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
00:28:33.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
00:28:33.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:33.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:33.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:33.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-1/logs
00:28:33.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-1/data/info.pb
00:28:33.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:33.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
00:28:33.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
00:28:33.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
00:28:33.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
00:28:33.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:33.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:33.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:33.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
00:28:33.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
00:28:33.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:28:33.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:28:33.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.175.253:43719
00:28:33.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.175.253
00:28:33.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46185
00:28:33.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/krb5kdc/kudu_127.7.175.253.keytab
00:28:33.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.175.253
00:28:33.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
00:28:33.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
00:28:33.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
00:28:33.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.175.212:41129
00:28:33.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:33.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:33.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:28:33.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:28:33.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.175.254:39253,127.7.175.253:43719,127.7.175.252:42903
00:28:33.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:28:33.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43589/jwks.json
00:28:33.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
00:28:33.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:33.723418 8301 sys_catalog.cc:263] Verifying existing consensus state
00:28:33.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:33.731312 8301 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2: Bootstrap starting.
00:28:33.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:33.762727 8301 log.cc:824] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2: Log is configured to *not* fsync() on all Append() calls
00:28:33.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:33.767424 8301 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
00:28:33.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:33.768229 8301 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2: Bootstrap complete.
00:28:33.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:33.802577 8301 raft_consensus.cc:348] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aad96a77b01c43088a1359442c9eaac6" member_type: VOTER last_known_addr { host: "127.7.175.254" port: 39253 } } peers { permanent_uuid: "a6b49e0918be4012bcf4db0751e4496f" member_type: VOTER last_known_addr { host: "127.7.175.253" port: 43719 } } peers { permanent_uuid: "4b3b2ea20cfb450d8ad78bdee67408e2" member_type: VOTER last_known_addr { host: "127.7.175.252" port: 42903 } }
00:28:33.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:33.805923 8301 raft_consensus.cc:374] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
00:28:33.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:33.806234 8301 raft_consensus.cc:729] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4b3b2ea20cfb450d8ad78bdee67408e2, State: Initialized, Role: FOLLOWER
00:28:33.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:33.807154 8301 consensus_queue.cc:260] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aad96a77b01c43088a1359442c9eaac6" member_type: VOTER last_known_addr { host: "127.7.175.254" port: 39253 } } peers { permanent_uuid: "a6b49e0918be4012bcf4db0751e4496f" member_type: VOTER last_known_addr { host: "127.7.175.253" port: 43719 } } peers { permanent_uuid: "4b3b2ea20cfb450d8ad78bdee67408e2" member_type: VOTER last_known_addr { host: "127.7.175.252" port: 42903 } }
00:28:33.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:33.810190 8305 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aad96a77b01c43088a1359442c9eaac6" member_type: VOTER last_known_addr { host: "127.7.175.254" port: 39253 } } peers { permanent_uuid: "a6b49e0918be4012bcf4db0751e4496f" member_type: VOTER last_known_addr { host: "127.7.175.253" port: 43719 } } peers { permanent_uuid: "4b3b2ea20cfb450d8ad78bdee67408e2" member_type: VOTER last_known_addr { host: "127.7.175.252" port: 42903 } } }
00:28:33.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:33.811466 8305 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [sys.catalog]: This master's current role is: FOLLOWER
00:28:33.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:33.811851 8301 sys_catalog.cc:564] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [sys.catalog]: configured and running, proceeding with master startup.
00:28:33.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:33.843744 8316 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:28:33.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:33.844367 8316 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:28:33.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:33.836447 8230 master_runner.cc:186] Error getting master registration for 127.7.175.254:39253: Network error: Client connection negotiation failed: client connection to 127.7.175.254:39253: connect: Connection refused (error 111),
00:28:33.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:33.864787 8230 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
00:28:33.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:33.872967 8305 raft_consensus.cc:482] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
00:28:33.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:33.873539 8305 raft_consensus.cc:504] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aad96a77b01c43088a1359442c9eaac6" member_type: VOTER last_known_addr { host: "127.7.175.254" port: 39253 } } peers { permanent_uuid: "a6b49e0918be4012bcf4db0751e4496f" member_type: VOTER last_known_addr { host: "127.7.175.253" port: 43719 } } peers { permanent_uuid: "4b3b2ea20cfb450d8ad78bdee67408e2" member_type: VOTER last_known_addr { host: "127.7.175.252" port: 42903 } }
00:28:33.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:33.876618 8305 leader_election.cc:288] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers aad96a77b01c43088a1359442c9eaac6 (127.7.175.254:39253), a6b49e0918be4012bcf4db0751e4496f (127.7.175.253:43719)
00:28:33.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:33.880359 8253 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.175.254:39253: connect: Connection refused (error 111)
00:28:33.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:33.885475 8253 leader_election.cc:334] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer aad96a77b01c43088a1359442c9eaac6 (127.7.175.254:39253): Network error: Client connection negotiation failed: client connection to 127.7.175.254:39253: connect: Connection refused (error 111)
00:28:33.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:33.885982 8253 leader_election.cc:334] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a6b49e0918be4012bcf4db0751e4496f (127.7.175.253:43719): Network error: Client connection negotiation failed: client connection to 127.7.175.253:43719: connect: Connection refused (error 111)
00:28:33.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:33.886281 8253 leader_election.cc:302] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4b3b2ea20cfb450d8ad78bdee67408e2; no voters: a6b49e0918be4012bcf4db0751e4496f, aad96a77b01c43088a1359442c9eaac6
00:28:33.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:33.886961 8305 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
00:28:33.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:33.891386 8255 tablet.cc:2369] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2: Can't schedule compaction. Clean time has not been advanced past its initial value.
00:28:34.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:34.068125 8303 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
00:28:34.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:34.068848 8303 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
00:28:34.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:34.069571 8303 flags.cc:425] Enabled unsafe flag: --never_fsync=true
00:28:34.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:34.102890 8303 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
00:28:34.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:34.103175 8303 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
00:28:34.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:34.103561 8303 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
00:28:34.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:34.103935 8303 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
00:28:34.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:34.104254 8303 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
00:28:34.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:34.104564 8303 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
00:28:34.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:34.104827 8303 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43589/jwks.json
00:28:34.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:34.144835 8303 master_runner.cc:386] Master server non-default flags:
00:28:34.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
00:28:34.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.175.212:41129
00:28:34.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
00:28:34.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
00:28:34.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-1/data
00:28:34.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-1/wal
00:28:34.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
00:28:34.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
00:28:34.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.175.254:39253,127.7.175.253:43719,127.7.175.252:42903
00:28:34.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/krb5kdc/kudu_127.7.175.253.keytab
00:28:34.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.175.253
00:28:34.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
00:28:34.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
00:28:34.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
00:28:34.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.175.253:43719
00:28:34.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
00:28:34.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
00:28:34.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
00:28:34.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43589/jwks.json
00:28:34.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
00:28:34.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
00:28:34.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
00:28:34.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
00:28:34.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
00:28:34.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-1/data/info.pb
00:28:34.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.175.253
00:28:34.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46185
00:28:34.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
00:28:34.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
00:28:34.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
00:28:34.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
00:28:34.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
00:28:34.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-1/logs
00:28:34.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
00:28:34.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
00:28:34.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
00:28:34.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
00:28:34.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
00:28:34.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
00:28:34.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
00:28:34.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 23 Sep 2026 23:32:10 UTC on ccycloud.quasar-wfxeao.root.comops.site
00:28:34.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4668917
00:28:34.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
00:28:34.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:34.146425 8303 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
00:28:34.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:34.147889 8303 file_cache.cc:493] Constructed file cache file cache with capacity 419430
00:28:34.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:34.169256 8327 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:34.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:34.850684 8316 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:28:34.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:34.851212 8316 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:28:35.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:35.348014 8297 debug-util.cc:398] Leaking SignalData structure 0x7b0800073060 after lost signal to thread 8231
00:28:35.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:35.348703 8297 debug-util.cc:398] Leaking SignalData structure 0x7b08000737c0 after lost signal to thread 8300
00:28:35.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:34.173247 8303 server_base.cc:1034] running on GCE node
00:28:35.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:34.173395 8329 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:35.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:34.170619 8326 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
00:28:35.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:35.475565 8303 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
00:28:35.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:35.478826 8303 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
00:28:35.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:35.482573 8303 hybrid_clock.cc:648] HybridClock initialized: now 1790209715482130 us; error 447 us; skew 500 ppm
00:28:35.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:35.484157 8303 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
00:28:35.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 00:28:35 dist-test-slave-dnss krb5kdc[7884](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790209715, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.175.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
00:28:35.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:35.493041 8303 init.cc:377] Logged in from keytab as kudu/127.7.175.253@KRBTEST.COM (short username kudu)
00:28:35.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:35.521256 8303 webserver.cc:460] Webserver started at http://127.7.175.253:46185/ using document root <none> and password file <none>
00:28:35.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:35.522804 8303 fs_manager.cc:359] Metadata directory not provided
00:28:35.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:35.524204 8303 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
00:28:35.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:35.540656 8303 fs_manager.cc:709] Time spent opening directory manager: real 0.012s user 0.006s sys 0.005s
00:28:35.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:35.553001 8339 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
00:28:35.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:35.554522 8303 fs_manager.cc:725] Time spent opening block manager: real 0.010s user 0.004s sys 0.004s
00:28:35.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:35.555068 8303 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-1/data,/tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-1/wal
00:28:35.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a6b49e0918be4012bcf4db0751e4496f"
00:28:35.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 00:28:27 on dist-test-slave-dnss"
00:28:35.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:35.559827 8303 fs_report.cc:383] FS layout report
00:28:35.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
00:28:35.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-1/wal
00:28:35.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-1/wal
00:28:35.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-1/data/data
00:28:35.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
00:28:35.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
00:28:35.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
00:28:35.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
00:28:35.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
00:28:35.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
00:28:35.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
00:28:35.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
00:28:35.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
00:28:35.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
00:28:35.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
00:28:35.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:35.642941 8303 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
00:28:35.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:35.652654 8303 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
00:28:35.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:35.653498 8303 kserver.cc:163] Server-wide thread pool size limit: 3276
00:28:35.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:35.829675 8303 rpc_server.cc:307] RPC server started. Bound to: 127.7.175.253:43719
00:28:35.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:35.829243 8390 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.175.253:43719 every 8 connection(s)
00:28:35.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:35.832901 8303 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-1/data/info.pb
00:28:35.836 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
00:28:35.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:35.834975 7871 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskNE0Lyu/build/tsan/bin/kudu as pid 8303
00:28:35.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:35.855760 8316 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:28:35.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:35.857003 8316 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:28:35.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:35.862545 8391 sys_catalog.cc:263] Verifying existing consensus state
00:28:35.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260924 00:28:35.863858 8391 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
00:28:35.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:35.865993 8303 master.cc:537] Master@127.7.175.253:43719 shutting down...
00:28:35.965 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.7.175.254:39253(127.7.175.254:39253)] connecting to peer
00:28:36.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:36.010157 8303 master.cc:552] Master@127.7.175.253:43719 shutdown complete.
00:28:36.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskNE0Lyu/test-tmp/mini-kudu-cluster2657927152153847687/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
00:28:36.057 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.7.175.252:42903(127.7.175.252:42903)] connecting to peer
00:28:36.060 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.7.175.253:43719(127.7.175.253:43719)] connecting to peer
00:28:36.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:36.111850 8396 raft_consensus.cc:482] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
00:28:36.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:36.112581 8396 raft_consensus.cc:504] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aad96a77b01c43088a1359442c9eaac6" member_type: VOTER last_known_addr { host: "127.7.175.254" port: 39253 } } peers { permanent_uuid: "a6b49e0918be4012bcf4db0751e4496f" member_type: VOTER last_known_addr { host: "127.7.175.253" port: 43719 } } peers { permanent_uuid: "4b3b2ea20cfb450d8ad78bdee67408e2" member_type: VOTER last_known_addr { host: "127.7.175.252" port: 42903 } }
00:28:36.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:36.114995 8396 leader_election.cc:288] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers aad96a77b01c43088a1359442c9eaac6 (127.7.175.254:39253), a6b49e0918be4012bcf4db0751e4496f (127.7.175.253:43719)
00:28:36.121 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.7.175.252:42903(127.7.175.252:42903)] Successfully connected to peer
00:28:36.124 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:28:36.128 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.175.254:39253(127.7.175.254:39253)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:28:36.136 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.175.254:39253: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:28:36.137 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:28:36.137 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.175.253:43719(127.7.175.253:43719)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:28:36.138 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.175.253:43719: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:28:36.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:36.139178 8253 leader_election.cc:334] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer aad96a77b01c43088a1359442c9eaac6 (127.7.175.254:39253): Network error: Client connection negotiation failed: client connection to 127.7.175.254:39253: connect: Connection refused (error 111)
00:28:36.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:36.139812 8253 leader_election.cc:334] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a6b49e0918be4012bcf4db0751e4496f (127.7.175.253:43719): Network error: Client connection negotiation failed: client connection to 127.7.175.253:43719: connect: Connection refused (error 111)
00:28:36.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:36.140151 8253 leader_election.cc:302] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4b3b2ea20cfb450d8ad78bdee67408e2; no voters: a6b49e0918be4012bcf4db0751e4496f, aad96a77b01c43088a1359442c9eaac6
00:28:36.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:36.140753 8396 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
00:28:36.217 [WARN - kudu-nio-1] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
00:28:36.225 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.7.175.252
00:28:36.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 00:28:36 dist-test-slave-dnss krb5kdc[7884](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790209709, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.7.175.252@KRBTEST.COM
00:28:36.506 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x93e48482, L:/127.0.0.1:34596 - R:/127.7.175.252:42903] using SASL/GSSAPI
00:28:36.526 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719; will retry
00:28:36.546 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.175.254:39253(127.7.175.254:39253)] connecting to peer
00:28:36.551 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.175.253:43719(127.7.175.253:43719)] connecting to peer
00:28:36.559 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:28:36.559 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.175.253:43719(127.7.175.253:43719)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:28:36.560 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.175.253:43719: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:28:36.564 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:28:36.564 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.175.254:39253(127.7.175.254:39253)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:28:36.565 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.175.254:39253: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:28:36.566 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719; will retry
00:28:36.585 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.175.254:39253(127.7.175.254:39253)] connecting to peer
00:28:36.587 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.175.253:43719(127.7.175.253:43719)] connecting to peer
00:28:36.589 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:28:36.589 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.175.253:43719(127.7.175.253:43719)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:28:36.590 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.175.253:43719: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:28:36.593 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:28:36.594 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.175.254:39253(127.7.175.254:39253)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:28:36.594 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.175.254:39253: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:28:36.595 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719; will retry
00:28:36.606 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.175.254:39253(127.7.175.254:39253)] connecting to peer
00:28:36.607 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.175.253:43719(127.7.175.253:43719)] connecting to peer
00:28:36.609 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:28:36.609 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:28:36.609 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.175.254:39253(127.7.175.254:39253)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:28:36.610 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.175.253:43719(127.7.175.253:43719)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:28:36.611 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.175.254:39253: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:28:36.611 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.175.253:43719: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:28:36.614 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719; will retry
00:28:36.646 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.175.254:39253(127.7.175.254:39253)] connecting to peer
00:28:36.648 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.175.253:43719(127.7.175.253:43719)] connecting to peer
00:28:36.650 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:28:36.650 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.175.253:43719(127.7.175.253:43719)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:28:36.651 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.175.253:43719: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:28:36.653 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:28:36.653 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.175.254:39253(127.7.175.254:39253)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:28:36.653 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.175.254:39253: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:28:36.655 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719; will retry
00:28:36.686 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.175.254:39253(127.7.175.254:39253)] connecting to peer
00:28:36.687 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.175.253:43719(127.7.175.253:43719)] connecting to peer
00:28:36.689 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:28:36.689 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.175.253:43719(127.7.175.253:43719)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:28:36.690 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.175.253:43719: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:28:36.697 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:28:36.697 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.175.254:39253(127.7.175.254:39253)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:28:36.697 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.175.254:39253: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:28:36.699 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719; will retry
00:28:36.726 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.175.254:39253(127.7.175.254:39253)] connecting to peer
00:28:36.728 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.175.253:43719(127.7.175.253:43719)] connecting to peer
00:28:36.730 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:28:36.731 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.175.253:43719(127.7.175.253:43719)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:28:36.731 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:28:36.731 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.175.254:39253(127.7.175.254:39253)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:28:36.732 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.175.253:43719: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:28:36.732 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.175.254:39253: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:28:36.733 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719; will retry
00:28:36.766 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.175.254:39253(127.7.175.254:39253)] connecting to peer
00:28:36.768 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.175.253:43719(127.7.175.253:43719)] connecting to peer
00:28:36.771 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:28:36.771 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.175.253:43719(127.7.175.253:43719)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:28:36.772 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.175.253:43719: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:28:36.773 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:28:36.773 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.175.254:39253(127.7.175.254:39253)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:28:36.775 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.175.254:39253: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:28:36.777 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719; will retry
00:28:36.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:36.860227 8316 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:28:36.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:36.860497 8316 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:28:36.965 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.175.254:39253(127.7.175.254:39253)] connecting to peer
00:28:36.967 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.175.253:43719(127.7.175.253:43719)] connecting to peer
00:28:36.970 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:28:36.970 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.175.253:43719(127.7.175.253:43719)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:28:36.973 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.175.253:43719: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:28:36.974 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:28:36.974 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.175.254:39253(127.7.175.254:39253)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:28:36.976 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.175.254:39253: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:28:36.977 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719; will retry
00:28:37.286 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.175.254:39253(127.7.175.254:39253)] connecting to peer
00:28:37.289 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.175.253:43719(127.7.175.253:43719)] connecting to peer
00:28:37.291 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:28:37.292 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.175.253:43719(127.7.175.253:43719)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:28:37.292 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.175.253:43719: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:28:37.293 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:28:37.293 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.175.254:39253(127.7.175.254:39253)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:28:37.294 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.175.254:39253: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:28:37.296 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719; will retry
00:28:37.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:37.863556 8316 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:28:37.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:37.863984 8316 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:28:37.906 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.175.254:39253(127.7.175.254:39253)] connecting to peer
00:28:37.907 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.175.253:43719(127.7.175.253:43719)] connecting to peer
00:28:37.909 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:28:37.909 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:28:37.910 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.175.254:39253(127.7.175.254:39253)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:28:37.910 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.175.253:43719(127.7.175.253:43719)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:28:37.910 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.175.253:43719: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:28:37.911 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.175.254:39253: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:28:37.912 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719; will retry
00:28:38.846 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.175.254:39253(127.7.175.254:39253)] connecting to peer
00:28:38.848 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.175.253:43719(127.7.175.253:43719)] connecting to peer
00:28:38.850 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:28:38.850 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:28:38.850 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.175.253:43719(127.7.175.253:43719)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:28:38.850 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.175.254:39253(127.7.175.254:39253)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:28:38.851 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.175.253:43719: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:28:38.851 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.175.254:39253: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:28:38.853 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719; will retry
00:28:38.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:38.855943 8404 raft_consensus.cc:482] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
00:28:38.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:38.856302 8404 raft_consensus.cc:504] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aad96a77b01c43088a1359442c9eaac6" member_type: VOTER last_known_addr { host: "127.7.175.254" port: 39253 } } peers { permanent_uuid: "a6b49e0918be4012bcf4db0751e4496f" member_type: VOTER last_known_addr { host: "127.7.175.253" port: 43719 } } peers { permanent_uuid: "4b3b2ea20cfb450d8ad78bdee67408e2" member_type: VOTER last_known_addr { host: "127.7.175.252" port: 42903 } }
00:28:38.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:38.857934 8404 leader_election.cc:288] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers aad96a77b01c43088a1359442c9eaac6 (127.7.175.254:39253), a6b49e0918be4012bcf4db0751e4496f (127.7.175.253:43719)
00:28:38.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:38.863703 8253 leader_election.cc:334] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer aad96a77b01c43088a1359442c9eaac6 (127.7.175.254:39253): Network error: Client connection negotiation failed: client connection to 127.7.175.254:39253: connect: Connection refused (error 111)
00:28:38.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:38.864398 8253 leader_election.cc:334] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a6b49e0918be4012bcf4db0751e4496f (127.7.175.253:43719): Network error: Client connection negotiation failed: client connection to 127.7.175.253:43719: connect: Connection refused (error 111)
00:28:38.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:38.864708 8253 leader_election.cc:302] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4b3b2ea20cfb450d8ad78bdee67408e2; no voters: a6b49e0918be4012bcf4db0751e4496f, aad96a77b01c43088a1359442c9eaac6
00:28:38.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:38.865554 8404 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
00:28:38.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:38.867394 8316 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:28:38.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:38.867815 8316 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:28:38.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:38.905643 8230 master_runner.cc:186] Error getting master registration for 127.7.175.254:39253: Network error: Client connection negotiation failed: client connection to 127.7.175.254:39253: connect: Connection refused (error 111),
00:28:38.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:38.905972 8230 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
00:28:39.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:39.871338 8316 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:28:39.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:39.871647 8316 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:28:40.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:40.874684 8316 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:28:40.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:40.874955 8316 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:28:41.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:41.878695 8316 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:28:41.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:41.879061 8316 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:28:42.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:42.159456 8412 raft_consensus.cc:482] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
00:28:42.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:42.160019 8412 raft_consensus.cc:504] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aad96a77b01c43088a1359442c9eaac6" member_type: VOTER last_known_addr { host: "127.7.175.254" port: 39253 } } peers { permanent_uuid: "a6b49e0918be4012bcf4db0751e4496f" member_type: VOTER last_known_addr { host: "127.7.175.253" port: 43719 } } peers { permanent_uuid: "4b3b2ea20cfb450d8ad78bdee67408e2" member_type: VOTER last_known_addr { host: "127.7.175.252" port: 42903 } }
00:28:42.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:42.162429 8412 leader_election.cc:288] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers aad96a77b01c43088a1359442c9eaac6 (127.7.175.254:39253), a6b49e0918be4012bcf4db0751e4496f (127.7.175.253:43719)
00:28:42.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:42.165382 8253 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.175.254:39253: connect: Connection refused (error 111) [suppressed 5 similar messages]
00:28:42.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:42.169453 8253 leader_election.cc:334] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer aad96a77b01c43088a1359442c9eaac6 (127.7.175.254:39253): Network error: Client connection negotiation failed: client connection to 127.7.175.254:39253: connect: Connection refused (error 111)
00:28:42.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:42.169898 8253 leader_election.cc:334] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a6b49e0918be4012bcf4db0751e4496f (127.7.175.253:43719): Network error: Client connection negotiation failed: client connection to 127.7.175.253:43719: connect: Connection refused (error 111)
00:28:42.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:42.170138 8253 leader_election.cc:302] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4b3b2ea20cfb450d8ad78bdee67408e2; no voters: a6b49e0918be4012bcf4db0751e4496f, aad96a77b01c43088a1359442c9eaac6
00:28:42.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:42.170771 8412 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
00:28:42.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:42.881850 8316 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:28:42.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:42.882128 8316 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:28:42.946 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.175.254:39253(127.7.175.254:39253)] connecting to peer
00:28:42.948 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.175.253:43719(127.7.175.253:43719)] connecting to peer
00:28:42.950 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:28:42.950 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.175.253:43719(127.7.175.253:43719)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:28:42.951 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.175.253:43719: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:28:42.951 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:28:42.951 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.175.254:39253(127.7.175.254:39253)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:28:42.951 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.175.254:39253: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:28:42.952 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719; will retry
00:28:43.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 00:28:43 dist-test-slave-dnss krb5kdc[7884](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790209723, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.175.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
00:28:43.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:43.569959 8245 init.cc:295] Successfully reacquired a new kerberos TGT
00:28:43.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:43.884945 8316 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:28:43.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:43.885245 8316 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:28:43.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:43.908787 8230 master_runner.cc:186] Error getting master registration for 127.7.175.254:39253: Network error: Client connection negotiation failed: client connection to 127.7.175.254:39253: connect: Connection refused (error 111),
00:28:43.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:43.909281 8230 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
00:28:44.026 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.175.254:39253(127.7.175.254:39253)] connecting to peer
00:28:44.027 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.175.253:43719(127.7.175.253:43719)] connecting to peer
00:28:44.030 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:28:44.030 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.175.253:43719(127.7.175.253:43719)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:28:44.030 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.175.253:43719: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:28:44.031 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:28:44.031 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.175.254:39253(127.7.175.254:39253)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:28:44.031 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.175.254:39253: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:28:44.033 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719; will retry
00:28:44.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:44.672672 7221 debug-util.cc:398] Leaking SignalData structure 0x7b080007df80 after lost signal to thread 7158
00:28:44.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:44.673216 7221 debug-util.cc:398] Leaking SignalData structure 0x7b0800097280 after lost signal to thread 7224
00:28:44.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:44.888090 8316 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:28:44.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:44.888384 8316 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:28:45.485 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.175.254:39253(127.7.175.254:39253)] connecting to peer
00:28:45.487 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.175.253:43719(127.7.175.253:43719)] connecting to peer
00:28:45.491 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:28:45.491 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.175.253:43719(127.7.175.253:43719)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:28:45.491 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:28:45.491 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.175.253:43719: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:28:45.491 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.175.254:39253(127.7.175.254:39253)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:28:45.491 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.175.254:39253: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:28:45.492 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719; will retry
00:28:45.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:45.891461 8316 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:28:45.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:45.891769 8316 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:28:46.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:46.895139 8316 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:28:46.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:46.895401 8316 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:28:47.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:47.022625 8420 raft_consensus.cc:482] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
00:28:47.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:47.022989 8420 raft_consensus.cc:504] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aad96a77b01c43088a1359442c9eaac6" member_type: VOTER last_known_addr { host: "127.7.175.254" port: 39253 } } peers { permanent_uuid: "a6b49e0918be4012bcf4db0751e4496f" member_type: VOTER last_known_addr { host: "127.7.175.253" port: 43719 } } peers { permanent_uuid: "4b3b2ea20cfb450d8ad78bdee67408e2" member_type: VOTER last_known_addr { host: "127.7.175.252" port: 42903 } }
00:28:47.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:47.024684 8420 leader_election.cc:288] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers aad96a77b01c43088a1359442c9eaac6 (127.7.175.254:39253), a6b49e0918be4012bcf4db0751e4496f (127.7.175.253:43719)
00:28:47.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:47.030117 8253 leader_election.cc:334] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer aad96a77b01c43088a1359442c9eaac6 (127.7.175.254:39253): Network error: Client connection negotiation failed: client connection to 127.7.175.254:39253: connect: Connection refused (error 111)
00:28:47.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:47.030684 8253 leader_election.cc:334] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a6b49e0918be4012bcf4db0751e4496f (127.7.175.253:43719): Network error: Client connection negotiation failed: client connection to 127.7.175.253:43719: connect: Connection refused (error 111)
00:28:47.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:47.030998 8253 leader_election.cc:302] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4b3b2ea20cfb450d8ad78bdee67408e2; no voters: a6b49e0918be4012bcf4db0751e4496f, aad96a77b01c43088a1359442c9eaac6
00:28:47.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:47.031606 8420 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
00:28:47.486 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.175.254:39253(127.7.175.254:39253)] connecting to peer
00:28:47.487 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.175.253:43719(127.7.175.253:43719)] connecting to peer
00:28:47.489 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:28:47.489 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:28:47.490 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.175.254:39253(127.7.175.254:39253)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:28:47.490 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.175.253:43719(127.7.175.253:43719)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:28:47.490 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.175.254:39253: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:28:47.490 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.175.253:43719: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:28:47.492 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719; will retry
00:28:47.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:47.898221 8316 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:28:47.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:47.898618 8316 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:28:48.006 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.175.254:39253(127.7.175.254:39253)] connecting to peer
00:28:48.008 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.175.253:43719(127.7.175.253:43719)] connecting to peer
00:28:48.010 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:28:48.010 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.175.254:39253(127.7.175.254:39253)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:28:48.010 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.175.254:39253: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:28:48.010 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:28:48.010 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.175.253:43719(127.7.175.253:43719)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:28:48.011 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.175.253:43719: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:28:48.012 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719; will retry
00:28:48.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:48.699903 7376 debug-util.cc:398] Leaking SignalData structure 0x7b08000bbba0 after lost signal to thread 7307
00:28:48.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:48.700578 7376 debug-util.cc:398] Leaking SignalData structure 0x7b08000ba4a0 after lost signal to thread 7379
00:28:48.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:48.902273 8316 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:28:48.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:48.902603 8316 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:28:48.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:48.912058 8230 master_runner.cc:186] Error getting master registration for 127.7.175.254:39253: Network error: Client connection negotiation failed: client connection to 127.7.175.254:39253: connect: Connection refused (error 111),
00:28:48.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:48.912545 8230 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
00:28:49.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:49.905308 8316 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:28:49.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:49.905592 8316 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:28:50.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:50.908497 8316 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:28:50.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:50.908808 8316 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:28:51.906 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.175.254:39253(127.7.175.254:39253)] connecting to peer
00:28:51.908 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.175.253:43719(127.7.175.253:43719)] connecting to peer
00:28:51.911 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:28:51.911 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.175.254:39253(127.7.175.254:39253)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:28:51.912 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.175.254:39253: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:28:51.913 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:28:51.913 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.175.253:43719(127.7.175.253:43719)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:28:51.914 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.175.253:43719: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:28:51.915 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719; will retry
00:28:51.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:51.915405 8316 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:28:51.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:51.916102 8316 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:28:52.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:52.587729 7715 debug-util.cc:398] Leaking SignalData structure 0x7b08000af260 after lost signal to thread 7589
00:28:52.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:52.588609 7715 debug-util.cc:398] Leaking SignalData structure 0x7b08000bec20 after lost signal to thread 7718
00:28:52.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:52.918989 8316 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:28:52.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:52.919246 8316 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:28:53.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 00:28:53 dist-test-slave-dnss krb5kdc[7884](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790209733, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.175.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
00:28:53.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:53.580053 8245 init.cc:295] Successfully reacquired a new kerberos TGT
00:28:53.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:53.654207 8434 raft_consensus.cc:482] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
00:28:53.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:53.654686 8434 raft_consensus.cc:504] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aad96a77b01c43088a1359442c9eaac6" member_type: VOTER last_known_addr { host: "127.7.175.254" port: 39253 } } peers { permanent_uuid: "a6b49e0918be4012bcf4db0751e4496f" member_type: VOTER last_known_addr { host: "127.7.175.253" port: 43719 } } peers { permanent_uuid: "4b3b2ea20cfb450d8ad78bdee67408e2" member_type: VOTER last_known_addr { host: "127.7.175.252" port: 42903 } }
00:28:53.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:53.656514 8434 leader_election.cc:288] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers aad96a77b01c43088a1359442c9eaac6 (127.7.175.254:39253), a6b49e0918be4012bcf4db0751e4496f (127.7.175.253:43719)
00:28:53.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:53.658951 8253 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.175.254:39253: connect: Connection refused (error 111) [suppressed 3 similar messages]
00:28:53.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:53.662024 8253 leader_election.cc:334] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer aad96a77b01c43088a1359442c9eaac6 (127.7.175.254:39253): Network error: Client connection negotiation failed: client connection to 127.7.175.254:39253: connect: Connection refused (error 111)
00:28:53.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:53.662633 8253 leader_election.cc:334] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a6b49e0918be4012bcf4db0751e4496f (127.7.175.253:43719): Network error: Client connection negotiation failed: client connection to 127.7.175.253:43719: connect: Connection refused (error 111)
00:28:53.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:53.662889 8253 leader_election.cc:302] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4b3b2ea20cfb450d8ad78bdee67408e2; no voters: a6b49e0918be4012bcf4db0751e4496f, aad96a77b01c43088a1359442c9eaac6
00:28:53.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:53.663421 8434 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
00:28:53.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:53.914975 8230 master_runner.cc:186] Error getting master registration for 127.7.175.254:39253: Network error: Client connection negotiation failed: client connection to 127.7.175.254:39253: connect: Connection refused (error 111),
00:28:53.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:53.915423 8230 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
00:28:53.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:53.922240 8316 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:28:53.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:53.922456 8316 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:28:54.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:54.925832 8316 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:28:54.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:54.926141 8316 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:28:55.565 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.175.254:39253(127.7.175.254:39253)] connecting to peer
00:28:55.567 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.175.253:43719(127.7.175.253:43719)] connecting to peer
00:28:55.568 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:28:55.569 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.175.254:39253(127.7.175.254:39253)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:28:55.569 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.175.254:39253: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:28:55.571 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:28:55.571 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.175.253:43719(127.7.175.253:43719)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:28:55.571 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.175.253:43719: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:28:55.572 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719; will retry
00:28:55.886 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.175.254:39253(127.7.175.254:39253)] connecting to peer
00:28:55.887 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.175.253:43719(127.7.175.253:43719)] connecting to peer
00:28:55.888 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:28:55.888 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:28:55.889 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.175.254:39253(127.7.175.254:39253)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:28:55.889 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.175.253:43719(127.7.175.253:43719)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:28:55.889 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.175.254:39253: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:28:55.889 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.175.253:43719: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:28:55.891 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719; will retry
00:28:55.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:55.929466 8316 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:28:55.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:55.929733 8316 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:28:56.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:56.465822 8442 raft_consensus.cc:482] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
00:28:56.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:56.466219 8442 raft_consensus.cc:504] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aad96a77b01c43088a1359442c9eaac6" member_type: VOTER last_known_addr { host: "127.7.175.254" port: 39253 } } peers { permanent_uuid: "a6b49e0918be4012bcf4db0751e4496f" member_type: VOTER last_known_addr { host: "127.7.175.253" port: 43719 } } peers { permanent_uuid: "4b3b2ea20cfb450d8ad78bdee67408e2" member_type: VOTER last_known_addr { host: "127.7.175.252" port: 42903 } }
00:28:56.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:56.468837 8442 leader_election.cc:288] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers aad96a77b01c43088a1359442c9eaac6 (127.7.175.254:39253), a6b49e0918be4012bcf4db0751e4496f (127.7.175.253:43719)
00:28:56.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:56.474184 8253 leader_election.cc:334] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer aad96a77b01c43088a1359442c9eaac6 (127.7.175.254:39253): Network error: Client connection negotiation failed: client connection to 127.7.175.254:39253: connect: Connection refused (error 111)
00:28:56.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:56.474776 8253 leader_election.cc:334] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a6b49e0918be4012bcf4db0751e4496f (127.7.175.253:43719): Network error: Client connection negotiation failed: client connection to 127.7.175.253:43719: connect: Connection refused (error 111)
00:28:56.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:56.475030 8253 leader_election.cc:302] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4b3b2ea20cfb450d8ad78bdee67408e2; no voters: a6b49e0918be4012bcf4db0751e4496f, aad96a77b01c43088a1359442c9eaac6
00:28:56.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:56.475536 8442 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
00:28:56.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:56.932399 8316 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:28:56.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:56.932690 8316 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:28:57.426 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.175.254:39253(127.7.175.254:39253)] connecting to peer
00:28:57.427 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.175.253:43719(127.7.175.253:43719)] connecting to peer
00:28:57.428 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:28:57.429 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.175.254:39253(127.7.175.254:39253)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:28:57.429 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.175.254:39253: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:28:57.429 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:28:57.429 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.175.253:43719(127.7.175.253:43719)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:28:57.430 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.175.253:43719: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:28:57.431 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719; will retry
00:28:57.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:57.935495 8316 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:28:57.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:57.935834 8316 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:28:58.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:58.917969 8230 master_runner.cc:186] Error getting master registration for 127.7.175.254:39253: Network error: Client connection negotiation failed: client connection to 127.7.175.254:39253: connect: Connection refused (error 111),
00:28:58.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:28:58.918332 8230 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
00:28:58.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:58.938647 8316 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:28:58.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:58.938949 8316 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:28:59.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:59.941764 8316 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:28:59.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:28:59.942078 8316 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:29:00.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:29:00.094555 8451 raft_consensus.cc:482] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
00:29:00.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:29:00.095041 8451 raft_consensus.cc:504] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "aad96a77b01c43088a1359442c9eaac6" member_type: VOTER last_known_addr { host: "127.7.175.254" port: 39253 } } peers { permanent_uuid: "a6b49e0918be4012bcf4db0751e4496f" member_type: VOTER last_known_addr { host: "127.7.175.253" port: 43719 } } peers { permanent_uuid: "4b3b2ea20cfb450d8ad78bdee67408e2" member_type: VOTER last_known_addr { host: "127.7.175.252" port: 42903 } }
00:29:00.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:29:00.096863 8451 leader_election.cc:288] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers aad96a77b01c43088a1359442c9eaac6 (127.7.175.254:39253), a6b49e0918be4012bcf4db0751e4496f (127.7.175.253:43719)
00:29:00.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:29:00.099244 8253 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.175.254:39253: connect: Connection refused (error 111) [suppressed 3 similar messages]
00:29:00.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:29:00.101979 8253 leader_election.cc:334] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer aad96a77b01c43088a1359442c9eaac6 (127.7.175.254:39253): Network error: Client connection negotiation failed: client connection to 127.7.175.254:39253: connect: Connection refused (error 111)
00:29:00.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:29:00.102578 8253 leader_election.cc:334] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a6b49e0918be4012bcf4db0751e4496f (127.7.175.253:43719): Network error: Client connection negotiation failed: client connection to 127.7.175.253:43719: connect: Connection refused (error 111)
00:29:00.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:29:00.102834 8253 leader_election.cc:302] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4b3b2ea20cfb450d8ad78bdee67408e2; no voters: a6b49e0918be4012bcf4db0751e4496f, aad96a77b01c43088a1359442c9eaac6
00:29:00.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:29:00.103384 8451 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
00:29:00.305 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.175.254:39253(127.7.175.254:39253)] connecting to peer
00:29:00.307 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.175.253:43719(127.7.175.253:43719)] connecting to peer
00:29:00.308 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:29:00.308 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:29:00.308 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.175.254:39253(127.7.175.254:39253)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:29:00.308 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.175.253:43719(127.7.175.253:43719)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:29:00.308 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.175.254:39253: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:29:00.308 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.175.253:43719: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:29:00.310 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719; will retry
00:29:00.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:29:00.944901 8316 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:29:00.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:29:00.945152 8316 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:29:01.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:29:01.948838 8316 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:29:01.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:29:01.949186 8316 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:29:02.446 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.175.254:39253(127.7.175.254:39253)] connecting to peer
00:29:02.447 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.175.253:43719(127.7.175.253:43719)] connecting to peer
00:29:02.448 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:29:02.448 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:29:02.448 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.175.253:43719(127.7.175.253:43719)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:29:02.448 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.175.254:39253(127.7.175.254:39253)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:29:02.448 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.175.253:43719: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:29:02.448 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.175.254:39253: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:29:02.451 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719; will retry
00:29:02.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:29:02.952056 8316 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4b3b2ea20cfb450d8ad78bdee67408e2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
00:29:02.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:29:02.952391 8316 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
00:29:03.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 00:29:03 dist-test-slave-dnss krb5kdc[7884](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790209743, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.175.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
00:29:03.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:29:03.582751 8245 init.cc:295] Successfully reacquired a new kerberos TGT
00:29:03.686 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.175.254:39253(127.7.175.254:39253)] connecting to peer
00:29:03.687 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.175.253:43719(127.7.175.253:43719)] connecting to peer
00:29:03.688 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:29:03.688 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:29:03.688 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.175.253:43719(127.7.175.253:43719)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:29:03.688 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.175.254:39253(127.7.175.254:39253)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:29:03.688 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.175.253:43719: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719
00:29:03.688 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.175.254:39253: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
00:29:03.691 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719; will retry
00:29:03.695 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=27849), Traces: [0ms] refreshing cache from master, [78ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.254:39253, [195ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.252:42903, [198ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.253:43719, [267ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.254:39253: Network error: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [275ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.253:43719: Network error: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [660ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.252:42903: OK, [665ms] delaying RPC due to: Service unavailable: Master config (127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [683ms] refreshing cache from master, [684ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.254:39253, [686ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.252:42903, [689ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.253:43719, [697ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.253:43719: Network error: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [701ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.252:42903: OK, [702ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.254:39253: Network error: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [704ms] delaying RPC due to: Service unavailable: Master config (127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [723ms] refreshing cache from master, [723ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.254:39253, [724ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.252:42903, [725ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.253:43719, [727ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.253:43719: Network error: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [731ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.252:42903: OK, [732ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.254:39253: Network error: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [733ms] delaying RPC due to: Service unavailable: Master config (127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [743ms] refreshing cache from master, [743ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.254:39253, [744ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.252:42903, [745ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.253:43719, [748ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.254:39253: Network error: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [748ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.253:43719: Network error: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [750ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.252:42903: OK, [753ms] delaying RPC due to: Service unavailable: Master config (127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [784ms] refreshing cache from master, [784ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.254:39253, [785ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.252:42903, [786ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.253:43719, [788ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.253:43719: Network error: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [791ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.254:39253: Network error: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [792ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.252:42903: OK, [793ms] delaying RPC due to: Service unavailable: Master config (127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [823ms] refreshing cache from master, [824ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.254:39253, [825ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.252:42903, [825ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.253:43719, [827ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.253:43719: Network error: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [834ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.252:42903: OK, [835ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.254:39253: Network error: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [837ms] delaying RPC due to: Service unavailable: Master config (127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [863ms] refreshing cache from master, [864ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.254:39253, [864ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.252:42903, [866ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.253:43719, [869ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.253:43719: Network error: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [869ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.254:39253: Network error: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [871ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.252:42903: OK, [872ms] delaying RPC due to: Service unavailable: Master config (127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [904ms] refreshing cache from master, [904ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.254:39253, [906ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.252:42903, [906ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.253:43719, [909ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.253:43719: Network error: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [911ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.254:39253: Network error: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [914ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.252:42903: OK, [915ms] delaying RPC due to: Service unavailable: Master config (127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [1103ms] refreshing cache from master, [1103ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.254:39253, [1104ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.252:42903, [1105ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.253:43719, [1108ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.253:43719: Network error: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [1112ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.252:42903: OK, [1113ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.254:39253: Network error: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [1115ms] delaying RPC due to: Service unavailable: Master config (127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [1423ms] refreshing cache from master, [1424ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.254:39253, [1425ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.252:42903, [1426ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.253:43719, [1430ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.253:43719: Network error: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [1431ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.254:39253: Network error: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [1433ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.252:42903: OK, [1434ms] delaying RPC due to: Service unavailable: Master config (127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [2043ms] refreshing cache from master, [2043ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.254:39253, [2044ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.252:42903, [2045ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.253:43719, [2048ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.253:43719: Network error: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [2049ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.254:39253: Network error: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [2050ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.252:42903: OK, [2051ms] delaying RPC due to: Service unavailable: Master config (127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [2984ms] refreshing cache from master, [2984ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.254:39253, [2985ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.252:42903, [2986ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.253:43719, [2988ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.253:43719: Network error: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [2989ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.254:39253: Network error: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [2990ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.252:42903: OK, [2991ms] delaying RPC due to: Service unavailable: Master config (127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [7083ms] refreshing cache from master, [7084ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.254:39253, [7085ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.252:42903, [7086ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.253:43719, [7088ms] too many traces: truncated at 100 traces, deferred=Deferred@1541798228(state=PENDING, result=null, callback=(continuation of Deferred@335348793 after retry RPC after error@771194746) -> (continuation of Deferred@691317786 after retry RPC after error@1567612721) -> (continuation of Deferred@650691689 after retry RPC after error@819094373) -> (continuation of Deferred@1007308057 after retry RPC after error@532651006) -> (continuation of Deferred@2134175724 after retry RPC after error@1549693476) -> (continuation of Deferred@852663859 after retry RPC after error@1372685308) -> (continuation of Deferred@163732901 after retry RPC after error@648204779) -> (continuation of Deferred@11005962 after retry RPC after error@1563712897) -> (continuation of Deferred@1182797047 after retry RPC after error@1990045765) -> (continuation of Deferred@1233463612 after retry RPC after error@916635096) -> (continuation of Deferred@1887692879 after retry RPC after error@1804038530) -> (continuation of Deferred@195782919 after retry RPC after error@683956695) -> (continuation of Deferred@1206521104 after retry RPC after error@389119758) -> (continuation of Deferred@1199755198 after retry RPC after error@1300376696) -> (continuation of Deferred@1519856213 after retry RPC after error@1187182342) -> (continuation of Deferred@392970079 after retry RPC after error@1272000550) -> (continuation of Deferred@678919819 after retry RPC after error@2036123204) -> (continuation of Deferred@264018167 after retry RPC after error@1294911782) -> (continuation of Deferred@1581151534 after retry RPC after error@1104148043) -> (continuation of Deferred@710915493 after retry RPC after error@293809713) -> (continuation of Deferred@1018801706 after retry RPC after error@41581144) -> (continuation of Deferred@1432290607 after retry RPC after error@1562729235) -> (continuation of Deferred@740062819 after retry RPC after error@2112184578), errback=(continuation of Deferred@335348793 after retry RPC after error@771194746) -> (continuation of Deferred@691317786 after retry RPC after error@1567612721) -> (continuation of Deferred@650691689 after retry RPC after error@819094373) -> (continuation of Deferred@1007308057 after retry RPC after error@532651006) -> (continuation of Deferred@2134175724 after retry RPC after error@1549693476) -> (continuation of Deferred@852663859 after retry RPC after error@1372685308) -> (continuation of Deferred@163732901 after retry RPC after error@648204779) -> (continuation of Deferred@11005962 after retry RPC after error@1563712897) -> (continuation of Deferred@1182797047 after retry RPC after error@1990045765) -> (continuation of Deferred@1233463612 after retry RPC after error@916635096) -> (continuation of Deferred@1887692879 after retry RPC after error@1804038530) -> (continuation of Deferred@195782919 after retry RPC after error@683956695) -> (continuation of Deferred@1206521104 after retry RPC after error@389119758) -> (continuation of Deferred@1199755198 after retry RPC after error@1300376696) -> (continuation of Deferred@1519856213 after retry RPC after error@1187182342) -> (continuation of Deferred@392970079 after retry RPC after error@1272000550) -> (continuation of Deferred@678919819 after retry RPC after error@2036123204) -> (continuation of Deferred@264018167 after retry RPC after error@1294911782) -> (continuation of Deferred@1581151534 after retry RPC after error@1104148043) -> (continuation of Deferred@710915493 after retry RPC after error@293809713) -> (continuation of Deferred@1018801706 after retry RPC after error@41581144) -> (continuation of Deferred@1432290607 after retry RPC after error@1562729235) -> (continuation of Deferred@740062819 after retry RPC after error@2112184578)))
00:29:03.699 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.7.175.252:42903(127.7.175.252:42903)] disconnecting while in state READY
00:29:03.700 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.7.175.254:39253(127.7.175.254:39253)] disconnecting while in state TERMINATED
00:29:03.701 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.7.175.253:43719(127.7.175.253:43719)] disconnecting while in state TERMINATED
00:29:03.705 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
00:29:03.707 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
00:29:03.709 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.7.175.252:42903(127.7.175.252:42903)] handling channelInactive
00:29:03.709 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.175.252:42903(127.7.175.252:42903)] cleaning up while in state READY due to: connection closed
00:29:03.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:29:03.711731 7871 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskNE0Lyu/build/tsan/bin/kudu with pid 8230
00:29:04.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-24T00:29:04Z chronyd exiting
00:29:05.959 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
00:29:05.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:29:05.960716 7145 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskNE0Lyu/build/tsan/bin/kudu with pid 7425
00:29:05.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:29:05.986987 7145 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskNE0Lyu/build/tsan/bin/kudu with pid 7585
00:29:06.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:29:06.020480 7145 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskNE0Lyu/build/tsan/bin/kudu with pid 7729
00:29:06.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:29:06.045819 7145 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskNE0Lyu/build/tsan/bin/kudu with pid 7157
00:29:06.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:29:06.076340 7145 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskNE0Lyu/build/tsan/bin/kudu with pid 7227
00:29:06.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:29:06.094129 7329 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.250.126:46847: connect: Connection refused (error 111)
00:29:06.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 00:29:06.097465 7329 consensus_peers.cc:487] T 00000000000000000000000000000000 P 739522ae64834cf0b7591777d2ccbd28 -> Peer 305bbef030aa4dbeb2c0bcd5718f0053 (127.6.250.126:46847): Couldn't send request to peer 305bbef030aa4dbeb2c0bcd5718f0053. Status: Network error: Client connection negotiation failed: client connection to 127.6.250.126:46847: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
00:29:06.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 00:29:06.108302 7145 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskNE0Lyu/build/tsan/bin/kudu with pid 7302
00:29:07.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-24T00:29:07Z chronyd exiting
00:29:08.303 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=27849), Traces: [0ms] refreshing cache from master, [78ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.254:39253, [195ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.252:42903, [198ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.253:43719, [267ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.254:39253: Network error: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [275ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.253:43719: Network error: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [660ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.252:42903: OK, [665ms] delaying RPC due to: Service unavailable: Master config (127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [683ms] refreshing cache from master, [684ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.254:39253, [686ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.252:42903, [689ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.253:43719, [697ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.253:43719: Network error: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [701ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.252:42903: OK, [702ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.254:39253: Network error: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [704ms] delaying RPC due to: Service unavailable: Master config (127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [723ms] refreshing cache from master, [723ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.254:39253, [724ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.252:42903, [725ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.253:43719, [727ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.253:43719: Network error: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [731ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.252:42903: OK, [732ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.254:39253: Network error: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [733ms] delaying RPC due to: Service unavailable: Master config (127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [743ms] refreshing cache from master, [743ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.254:39253, [744ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.252:42903, [745ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.253:43719, [748ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.254:39253: Network error: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [748ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.253:43719: Network error: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [750ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.252:42903: OK, [753ms] delaying RPC due to: Service unavailable: Master config (127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [784ms] refreshing cache from master, [784ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.254:39253, [785ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.252:42903, [786ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.253:43719, [788ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.253:43719: Network error: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [791ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.254:39253: Network error: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [792ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.252:42903: OK, [793ms] delaying RPC due to: Service unavailable: Master config (127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [823ms] refreshing cache from master, [824ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.254:39253, [825ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.252:42903, [825ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.253:43719, [827ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.253:43719: Network error: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [834ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.252:42903: OK, [835ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.254:39253: Network error: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [837ms] delaying RPC due to: Service unavailable: Master config (127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [863ms] refreshing cache from master, [864ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.254:39253, [864ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.252:42903, [866ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.253:43719, [869ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.253:43719: Network error: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [869ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.254:39253: Network error: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [871ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.252:42903: OK, [872ms] delaying RPC due to: Service unavailable: Master config (127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [904ms] refreshing cache from master, [904ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.254:39253, [906ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.252:42903, [906ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.253:43719, [909ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.253:43719: Network error: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [911ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.254:39253: Network error: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [914ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.252:42903: OK, [915ms] delaying RPC due to: Service unavailable: Master config (127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [1103ms] refreshing cache from master, [1103ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.254:39253, [1104ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.252:42903, [1105ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.253:43719, [1108ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.253:43719: Network error: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [1112ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.252:42903: OK, [1113ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.254:39253: Network error: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [1115ms] delaying RPC due to: Service unavailable: Master config (127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [1423ms] refreshing cache from master, [1424ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.254:39253, [1425ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.252:42903, [1426ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.253:43719, [1430ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.253:43719: Network error: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [1431ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.254:39253: Network error: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [1433ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.252:42903: OK, [1434ms] delaying RPC due to: Service unavailable: Master config (127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [2043ms] refreshing cache from master, [2043ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.254:39253, [2044ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.252:42903, [2045ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.253:43719, [2048ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.253:43719: Network error: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [2049ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.254:39253: Network error: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [2050ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.252:42903: OK, [2051ms] delaying RPC due to: Service unavailable: Master config (127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [2984ms] refreshing cache from master, [2984ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.254:39253, [2985ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.252:42903, [2986ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.253:43719, [2988ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.253:43719: Network error: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [2989ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.254:39253: Network error: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [2990ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.252:42903: OK, [2991ms] delaying RPC due to: Service unavailable: Master config (127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [7083ms] refreshing cache from master, [7084ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.254:39253, [7085ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.252:42903, [7086ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.253:43719, [7088ms] too many traces: truncated at 100 traces, deferred=Deferred@1541798228(state=PENDING, result=null, callback=(continuation of Deferred@335348793 after retry RPC after error@771194746) -> (continuation of Deferred@691317786 after retry RPC after error@1567612721) -> (continuation of Deferred@650691689 after retry RPC after error@819094373) -> (continuation of Deferred@1007308057 after retry RPC after error@532651006) -> (continuation of Deferred@2134175724 after retry RPC after error@1549693476) -> (continuation of Deferred@852663859 after retry RPC after error@1372685308) -> (continuation of Deferred@163732901 after retry RPC after error@648204779) -> (continuation of Deferred@11005962 after retry RPC after error@1563712897) -> (continuation of Deferred@1182797047 after retry RPC after error@1990045765) -> (continuation of Deferred@1233463612 after retry RPC after error@916635096) -> (continuation of Deferred@1887692879 after retry RPC after error@1804038530) -> (continuation of Deferred@195782919 after retry RPC after error@683956695) -> (continuation of Deferred@1206521104 after retry RPC after error@389119758) -> (continuation of Deferred@1199755198 after retry RPC after error@1300376696) -> (continuation of Deferred@1519856213 after retry RPC after error@1187182342) -> (continuation of Deferred@392970079 after retry RPC after error@1272000550) -> (continuation of Deferred@678919819 after retry RPC after error@2036123204) -> (continuation of Deferred@264018167 after retry RPC after error@1294911782) -> (continuation of Deferred@1581151534 after retry RPC after error@1104148043) -> (continuation of Deferred@710915493 after retry RPC after error@293809713) -> (continuation of Deferred@1018801706 after retry RPC after error@41581144) -> (continuation of Deferred@1432290607 after retry RPC after error@1562729235) -> (continuation of Deferred@740062819 after retry RPC after error@2112184578), errback=(continuation of Deferred@335348793 after retry RPC after error@771194746) -> (continuation of Deferred@691317786 after retry RPC after error@1567612721) -> (continuation of Deferred@650691689 after retry RPC after error@819094373) -> (continuation of Deferred@1007308057 after retry RPC after error@532651006) -> (continuation of Deferred@2134175724 after retry RPC after error@1549693476) -> (continuation of Deferred@852663859 after retry RPC after error@1372685308) -> (continuation of Deferred@163732901 after retry RPC after error@648204779) -> (continuation of Deferred@11005962 after retry RPC after error@1563712897) -> (continuation of Deferred@1182797047 after retry RPC after error@1990045765) -> (continuation of Deferred@1233463612 after retry RPC after error@916635096) -> (continuation of Deferred@1887692879 after retry RPC after error@1804038530) -> (continuation of Deferred@195782919 after retry RPC after error@683956695) -> (continuation of Deferred@1206521104 after retry RPC after error@389119758) -> (continuation of Deferred@1199755198 after retry RPC after error@1300376696) -> (continuation of Deferred@1519856213 after retry RPC after error@1187182342) -> (continuation of Deferred@392970079 after retry RPC after error@1272000550) -> (continuation of Deferred@678919819 after retry RPC after error@2036123204) -> (continuation of Deferred@264018167 after retry RPC after error@1294911782) -> (continuation of Deferred@1581151534 after retry RPC after error@1104148043) -> (continuation of Deferred@710915493 after retry RPC after error@293809713) -> (continuation of Deferred@1018801706 after retry RPC after error@41581144) -> (continuation of Deferred@1432290607 after retry RPC after error@1562729235) -> (continuation of Deferred@740062819 after retry RPC after error@2112184578)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.7.175.254:39253
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=27849), Traces: [0ms] refreshing cache from master, [78ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.254:39253, [195ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.252:42903, [198ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.253:43719, [267ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.254:39253: Network error: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [275ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.253:43719: Network error: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [660ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.252:42903: OK, [665ms] delaying RPC due to: Service unavailable: Master config (127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [683ms] refreshing cache from master, [684ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.254:39253, [686ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.252:42903, [689ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.253:43719, [697ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.253:43719: Network error: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [701ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.252:42903: OK, [702ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.254:39253: Network error: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [704ms] delaying RPC due to: Service unavailable: Master config (127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [723ms] refreshing cache from master, [723ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.254:39253, [724ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.252:42903, [725ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.253:43719, [727ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.253:43719: Network error: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [731ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.252:42903: OK, [732ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.254:39253: Network error: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [733ms] delaying RPC due to: Service unavailable: Master config (127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [743ms] refreshing cache from master, [743ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.254:39253, [744ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.252:42903, [745ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.253:43719, [748ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.254:39253: Network error: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [748ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.253:43719: Network error: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [750ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.252:42903: OK, [753ms] delaying RPC due to: Service unavailable: Master config (127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [784ms] refreshing cache from master, [784ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.254:39253, [785ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.252:42903, [786ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.253:43719, [788ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.253:43719: Network error: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [791ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.254:39253: Network error: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [792ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.252:42903: OK, [793ms] delaying RPC due to: Service unavailable: Master config (127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [823ms] refreshing cache from master, [824ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.254:39253, [825ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.252:42903, [825ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.253:43719, [827ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.253:43719: Network error: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [834ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.252:42903: OK, [835ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.254:39253: Network error: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [837ms] delaying RPC due to: Service unavailable: Master config (127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [863ms] refreshing cache from master, [864ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.254:39253, [864ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.252:42903, [866ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.253:43719, [869ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.253:43719: Network error: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [869ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.254:39253: Network error: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [871ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.252:42903: OK, [872ms] delaying RPC due to: Service unavailable: Master config (127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [904ms] refreshing cache from master, [904ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.254:39253, [906ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.252:42903, [906ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.253:43719, [909ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.253:43719: Network error: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [911ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.254:39253: Network error: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [914ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.252:42903: OK, [915ms] delaying RPC due to: Service unavailable: Master config (127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [1103ms] refreshing cache from master, [1103ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.254:39253, [1104ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.252:42903, [1105ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.253:43719, [1108ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.253:43719: Network error: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [1112ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.252:42903: OK, [1113ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.254:39253: Network error: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [1115ms] delaying RPC due to: Service unavailable: Master config (127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [1423ms] refreshing cache from master, [1424ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.254:39253, [1425ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.252:42903, [1426ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.253:43719, [1430ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.253:43719: Network error: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [1431ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.254:39253: Network error: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [1433ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.252:42903: OK, [1434ms] delaying RPC due to: Service unavailable: Master config (127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [2043ms] refreshing cache from master, [2043ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.254:39253, [2044ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.252:42903, [2045ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.253:43719, [2048ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.253:43719: Network error: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [2049ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.254:39253: Network error: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [2050ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.252:42903: OK, [2051ms] delaying RPC due to: Service unavailable: Master config (127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [2984ms] refreshing cache from master, [2984ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.254:39253, [2985ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.252:42903, [2986ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.253:43719, [2988ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.253:43719: Network error: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [2989ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.254:39253: Network error: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [2990ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.252:42903: OK, [2991ms] delaying RPC due to: Service unavailable: Master config (127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [7083ms] refreshing cache from master, [7084ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.254:39253, [7085ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.252:42903, [7086ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.253:43719, [7088ms] too many traces: truncated at 100 traces, deferred=Deferred@1541798228(state=PENDING, result=null, callback=(continuation of Deferred@335348793 after retry RPC after error@771194746) -> (continuation of Deferred@691317786 after retry RPC after error@1567612721) -> (continuation of Deferred@650691689 after retry RPC after error@819094373) -> (continuation of Deferred@1007308057 after retry RPC after error@532651006) -> (continuation of Deferred@2134175724 after retry RPC after error@1549693476) -> (continuation of Deferred@852663859 after retry RPC after error@1372685308) -> (continuation of Deferred@163732901 after retry RPC after error@648204779) -> (continuation of Deferred@11005962 after retry RPC after error@1563712897) -> (continuation of Deferred@1182797047 after retry RPC after error@1990045765) -> (continuation of Deferred@1233463612 after retry RPC after error@916635096) -> (continuation of Deferred@1887692879 after retry RPC after error@1804038530) -> (continuation of Deferred@195782919 after retry RPC after error@683956695) -> (continuation of Deferred@1206521104 after retry RPC after error@389119758) -> (continuation of Deferred@1199755198 after retry RPC after error@1300376696) -> (continuation of Deferred@1519856213 after retry RPC after error@1187182342) -> (continuation of Deferred@392970079 after retry RPC after error@1272000550) -> (continuation of Deferred@678919819 after retry RPC after error@2036123204) -> (continuation of Deferred@264018167 after retry RPC after error@1294911782) -> (continuation of Deferred@1581151534 after retry RPC after error@1104148043) -> (continuation of Deferred@710915493 after retry RPC after error@293809713) -> (continuation of Deferred@1018801706 after retry RPC after error@41581144) -> (continuation of Deferred@1432290607 after retry RPC after error@1562729235) -> (continuation of Deferred@740062819 after retry RPC after error@2112184578), errback=(continuation of Deferred@335348793 after retry RPC after error@771194746) -> (continuation of Deferred@691317786 after retry RPC after error@1567612721) -> (continuation of Deferred@650691689 after retry RPC after error@819094373) -> (continuation of Deferred@1007308057 after retry RPC after error@532651006) -> (continuation of Deferred@2134175724 after retry RPC after error@1549693476) -> (continuation of Deferred@852663859 after retry RPC after error@1372685308) -> (continuation of Deferred@163732901 after retry RPC after error@648204779) -> (continuation of Deferred@11005962 after retry RPC after error@1563712897) -> (continuation of Deferred@1182797047 after retry RPC after error@1990045765) -> (continuation of Deferred@1233463612 after retry RPC after error@916635096) -> (continuation of Deferred@1887692879 after retry RPC after error@1804038530) -> (continuation of Deferred@195782919 after retry RPC after error@683956695) -> (continuation of Deferred@1206521104 after retry RPC after error@389119758) -> (continuation of Deferred@1199755198 after retry RPC after error@1300376696) -> (continuation of Deferred@1519856213 after retry RPC after error@1187182342) -> (continuation of Deferred@392970079 after retry RPC after error@1272000550) -> (continuation of Deferred@678919819 after retry RPC after error@2036123204) -> (continuation of Deferred@264018167 after retry RPC after error@1294911782) -> (continuation of Deferred@1581151534 after retry RPC after error@1104148043) -> (continuation of Deferred@710915493 after retry RPC after error@293809713) -> (continuation of Deferred@1018801706 after retry RPC after error@41581144) -> (continuation of Deferred@1432290607 after retry RPC after error@1562729235) -> (continuation of Deferred@740062819 after retry RPC after error@2112184578)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.7.175.254:39253
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
00:29:08.709 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
00:29:08.709 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=27849), Traces: [0ms] refreshing cache from master, [78ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.254:39253, [195ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.252:42903, [198ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.253:43719, [267ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.254:39253: Network error: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [275ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.253:43719: Network error: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [660ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.252:42903: OK, [665ms] delaying RPC due to: Service unavailable: Master config (127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [683ms] refreshing cache from master, [684ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.254:39253, [686ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.252:42903, [689ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.253:43719, [697ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.253:43719: Network error: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [701ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.252:42903: OK, [702ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.254:39253: Network error: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [704ms] delaying RPC due to: Service unavailable: Master config (127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [723ms] refreshing cache from master, [723ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.254:39253, [724ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.252:42903, [725ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.253:43719, [727ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.253:43719: Network error: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [731ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.252:42903: OK, [732ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.254:39253: Network error: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [733ms] delaying RPC due to: Service unavailable: Master config (127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [743ms] refreshing cache from master, [743ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.254:39253, [744ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.252:42903, [745ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.253:43719, [748ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.254:39253: Network error: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [748ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.253:43719: Network error: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [750ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.252:42903: OK, [753ms] delaying RPC due to: Service unavailable: Master config (127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [784ms] refreshing cache from master, [784ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.254:39253, [785ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.252:42903, [786ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.253:43719, [788ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.253:43719: Network error: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [791ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.254:39253: Network error: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [792ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.252:42903: OK, [793ms] delaying RPC due to: Service unavailable: Master config (127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [823ms] refreshing cache from master, [824ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.254:39253, [825ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.252:42903, [825ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.253:43719, [827ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.253:43719: Network error: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [834ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.252:42903: OK, [835ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.254:39253: Network error: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [837ms] delaying RPC due to: Service unavailable: Master config (127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [863ms] refreshing cache from master, [864ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.254:39253, [864ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.252:42903, [866ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.253:43719, [869ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.253:43719: Network error: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [869ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.254:39253: Network error: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [871ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.252:42903: OK, [872ms] delaying RPC due to: Service unavailable: Master config (127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [904ms] refreshing cache from master, [904ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.254:39253, [906ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.252:42903, [906ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.253:43719, [909ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.253:43719: Network error: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [911ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.254:39253: Network error: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [914ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.252:42903: OK, [915ms] delaying RPC due to: Service unavailable: Master config (127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [1103ms] refreshing cache from master, [1103ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.254:39253, [1104ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.252:42903, [1105ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.253:43719, [1108ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.253:43719: Network error: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [1112ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.252:42903: OK, [1113ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.254:39253: Network error: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [1115ms] delaying RPC due to: Service unavailable: Master config (127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [1423ms] refreshing cache from master, [1424ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.254:39253, [1425ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.252:42903, [1426ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.253:43719, [1430ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.253:43719: Network error: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [1431ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.254:39253: Network error: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [1433ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.252:42903: OK, [1434ms] delaying RPC due to: Service unavailable: Master config (127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [2043ms] refreshing cache from master, [2043ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.254:39253, [2044ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.252:42903, [2045ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.253:43719, [2048ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.253:43719: Network error: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [2049ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.254:39253: Network error: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [2050ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.252:42903: OK, [2051ms] delaying RPC due to: Service unavailable: Master config (127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [2984ms] refreshing cache from master, [2984ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.254:39253, [2985ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.252:42903, [2986ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.253:43719, [2988ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.253:43719: Network error: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [2989ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.254:39253: Network error: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [2990ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.252:42903: OK, [2991ms] delaying RPC due to: Service unavailable: Master config (127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [7083ms] refreshing cache from master, [7084ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.254:39253, [7085ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.252:42903, [7086ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.253:43719, [7088ms] too many traces: truncated at 100 traces, deferred=Deferred@1541798228(state=PENDING, result=null, callback=(continuation of Deferred@335348793 after retry RPC after error@771194746) -> (continuation of Deferred@691317786 after retry RPC after error@1567612721) -> (continuation of Deferred@650691689 after retry RPC after error@819094373) -> (continuation of Deferred@1007308057 after retry RPC after error@532651006) -> (continuation of Deferred@2134175724 after retry RPC after error@1549693476) -> (continuation of Deferred@852663859 after retry RPC after error@1372685308) -> (continuation of Deferred@163732901 after retry RPC after error@648204779) -> (continuation of Deferred@11005962 after retry RPC after error@1563712897) -> (continuation of Deferred@1182797047 after retry RPC after error@1990045765) -> (continuation of Deferred@1233463612 after retry RPC after error@916635096) -> (continuation of Deferred@1887692879 after retry RPC after error@1804038530) -> (continuation of Deferred@195782919 after retry RPC after error@683956695) -> (continuation of Deferred@1206521104 after retry RPC after error@389119758) -> (continuation of Deferred@1199755198 after retry RPC after error@1300376696) -> (continuation of Deferred@1519856213 after retry RPC after error@1187182342) -> (continuation of Deferred@392970079 after retry RPC after error@1272000550) -> (continuation of Deferred@678919819 after retry RPC after error@2036123204) -> (continuation of Deferred@264018167 after retry RPC after error@1294911782) -> (continuation of Deferred@1581151534 after retry RPC after error@1104148043) -> (continuation of Deferred@710915493 after retry RPC after error@293809713) -> (continuation of Deferred@1018801706 after retry RPC after error@41581144) -> (continuation of Deferred@1432290607 after retry RPC after error@1562729235) -> (continuation of Deferred@740062819 after retry RPC after error@2112184578), errback=(continuation of Deferred@335348793 after retry RPC after error@771194746) -> (continuation of Deferred@691317786 after retry RPC after error@1567612721) -> (continuation of Deferred@650691689 after retry RPC after error@819094373) -> (continuation of Deferred@1007308057 after retry RPC after error@532651006) -> (continuation of Deferred@2134175724 after retry RPC after error@1549693476) -> (continuation of Deferred@852663859 after retry RPC after error@1372685308) -> (continuation of Deferred@163732901 after retry RPC after error@648204779) -> (continuation of Deferred@11005962 after retry RPC after error@1563712897) -> (continuation of Deferred@1182797047 after retry RPC after error@1990045765) -> (continuation of Deferred@1233463612 after retry RPC after error@916635096) -> (continuation of Deferred@1887692879 after retry RPC after error@1804038530) -> (continuation of Deferred@195782919 after retry RPC after error@683956695) -> (continuation of Deferred@1206521104 after retry RPC after error@389119758) -> (continuation of Deferred@1199755198 after retry RPC after error@1300376696) -> (continuation of Deferred@1519856213 after retry RPC after error@1187182342) -> (continuation of Deferred@392970079 after retry RPC after error@1272000550) -> (continuation of Deferred@678919819 after retry RPC after error@2036123204) -> (continuation of Deferred@264018167 after retry RPC after error@1294911782) -> (continuation of Deferred@1581151534 after retry RPC after error@1104148043) -> (continuation of Deferred@710915493 after retry RPC after error@293809713) -> (continuation of Deferred@1018801706 after retry RPC after error@41581144) -> (continuation of Deferred@1432290607 after retry RPC after error@1562729235) -> (continuation of Deferred@740062819 after retry RPC after error@2112184578)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.7.175.254:39253
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=27849), Traces: [0ms] refreshing cache from master, [78ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.254:39253, [195ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.252:42903, [198ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.253:43719, [267ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.254:39253: Network error: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [275ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.253:43719: Network error: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [660ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.252:42903: OK, [665ms] delaying RPC due to: Service unavailable: Master config (127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [683ms] refreshing cache from master, [684ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.254:39253, [686ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.252:42903, [689ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.253:43719, [697ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.253:43719: Network error: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [701ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.252:42903: OK, [702ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.254:39253: Network error: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [704ms] delaying RPC due to: Service unavailable: Master config (127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [723ms] refreshing cache from master, [723ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.254:39253, [724ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.252:42903, [725ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.253:43719, [727ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.253:43719: Network error: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [731ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.252:42903: OK, [732ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.254:39253: Network error: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [733ms] delaying RPC due to: Service unavailable: Master config (127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [743ms] refreshing cache from master, [743ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.254:39253, [744ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.252:42903, [745ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.253:43719, [748ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.254:39253: Network error: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [748ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.253:43719: Network error: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [750ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.252:42903: OK, [753ms] delaying RPC due to: Service unavailable: Master config (127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [784ms] refreshing cache from master, [784ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.254:39253, [785ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.252:42903, [786ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.253:43719, [788ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.253:43719: Network error: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [791ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.254:39253: Network error: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [792ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.252:42903: OK, [793ms] delaying RPC due to: Service unavailable: Master config (127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [823ms] refreshing cache from master, [824ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.254:39253, [825ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.252:42903, [825ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.253:43719, [827ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.253:43719: Network error: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [834ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.252:42903: OK, [835ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.254:39253: Network error: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [837ms] delaying RPC due to: Service unavailable: Master config (127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [863ms] refreshing cache from master, [864ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.254:39253, [864ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.252:42903, [866ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.253:43719, [869ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.253:43719: Network error: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [869ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.254:39253: Network error: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [871ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.252:42903: OK, [872ms] delaying RPC due to: Service unavailable: Master config (127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [904ms] refreshing cache from master, [904ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.254:39253, [906ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.252:42903, [906ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.253:43719, [909ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.253:43719: Network error: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [911ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.254:39253: Network error: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [914ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.252:42903: OK, [915ms] delaying RPC due to: Service unavailable: Master config (127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [1103ms] refreshing cache from master, [1103ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.254:39253, [1104ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.252:42903, [1105ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.253:43719, [1108ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.253:43719: Network error: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [1112ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.252:42903: OK, [1113ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.254:39253: Network error: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [1115ms] delaying RPC due to: Service unavailable: Master config (127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [1423ms] refreshing cache from master, [1424ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.254:39253, [1425ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.252:42903, [1426ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.253:43719, [1430ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.253:43719: Network error: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [1431ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.254:39253: Network error: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [1433ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.252:42903: OK, [1434ms] delaying RPC due to: Service unavailable: Master config (127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [2043ms] refreshing cache from master, [2043ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.254:39253, [2044ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.252:42903, [2045ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.253:43719, [2048ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.253:43719: Network error: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [2049ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.254:39253: Network error: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [2050ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.252:42903: OK, [2051ms] delaying RPC due to: Service unavailable: Master config (127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [2984ms] refreshing cache from master, [2984ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.254:39253, [2985ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.252:42903, [2986ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.253:43719, [2988ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.253:43719: Network error: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719, [2989ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.254:39253: Network error: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [2990ms] Sub RPC ConnectToMaster: received response from server master-127.7.175.252:42903: OK, [2991ms] delaying RPC due to: Service unavailable: Master config (127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253, [7083ms] refreshing cache from master, [7084ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.254:39253, [7085ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.252:42903, [7086ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.175.253:43719, [7088ms] too many traces: truncated at 100 traces, deferred=Deferred@1541798228(state=PENDING, result=null, callback=(continuation of Deferred@335348793 after retry RPC after error@771194746) -> (continuation of Deferred@691317786 after retry RPC after error@1567612721) -> (continuation of Deferred@650691689 after retry RPC after error@819094373) -> (continuation of Deferred@1007308057 after retry RPC after error@532651006) -> (continuation of Deferred@2134175724 after retry RPC after error@1549693476) -> (continuation of Deferred@852663859 after retry RPC after error@1372685308) -> (continuation of Deferred@163732901 after retry RPC after error@648204779) -> (continuation of Deferred@11005962 after retry RPC after error@1563712897) -> (continuation of Deferred@1182797047 after retry RPC after error@1990045765) -> (continuation of Deferred@1233463612 after retry RPC after error@916635096) -> (continuation of Deferred@1887692879 after retry RPC after error@1804038530) -> (continuation of Deferred@195782919 after retry RPC after error@683956695) -> (continuation of Deferred@1206521104 after retry RPC after error@389119758) -> (continuation of Deferred@1199755198 after retry RPC after error@1300376696) -> (continuation of Deferred@1519856213 after retry RPC after error@1187182342) -> (continuation of Deferred@392970079 after retry RPC after error@1272000550) -> (continuation of Deferred@678919819 after retry RPC after error@2036123204) -> (continuation of Deferred@264018167 after retry RPC after error@1294911782) -> (continuation of Deferred@1581151534 after retry RPC after error@1104148043) -> (continuation of Deferred@710915493 after retry RPC after error@293809713) -> (continuation of Deferred@1018801706 after retry RPC after error@41581144) -> (continuation of Deferred@1432290607 after retry RPC after error@1562729235) -> (continuation of Deferred@740062819 after retry RPC after error@2112184578), errback=(continuation of Deferred@335348793 after retry RPC after error@771194746) -> (continuation of Deferred@691317786 after retry RPC after error@1567612721) -> (continuation of Deferred@650691689 after retry RPC after error@819094373) -> (continuation of Deferred@1007308057 after retry RPC after error@532651006) -> (continuation of Deferred@2134175724 after retry RPC after error@1549693476) -> (continuation of Deferred@852663859 after retry RPC after error@1372685308) -> (continuation of Deferred@163732901 after retry RPC after error@648204779) -> (continuation of Deferred@11005962 after retry RPC after error@1563712897) -> (continuation of Deferred@1182797047 after retry RPC after error@1990045765) -> (continuation of Deferred@1233463612 after retry RPC after error@916635096) -> (continuation of Deferred@1887692879 after retry RPC after error@1804038530) -> (continuation of Deferred@195782919 after retry RPC after error@683956695) -> (continuation of Deferred@1206521104 after retry RPC after error@389119758) -> (continuation of Deferred@1199755198 after retry RPC after error@1300376696) -> (continuation of Deferred@1519856213 after retry RPC after error@1187182342) -> (continuation of Deferred@392970079 after retry RPC after error@1272000550) -> (continuation of Deferred@678919819 after retry RPC after error@2036123204) -> (continuation of Deferred@264018167 after retry RPC after error@1294911782) -> (continuation of Deferred@1581151534 after retry RPC after error@1104148043) -> (continuation of Deferred@710915493 after retry RPC after error@293809713) -> (continuation of Deferred@1018801706 after retry RPC after error@41581144) -> (continuation of Deferred@1432290607 after retry RPC after error@1562729235) -> (continuation of Deferred@740062819 after retry RPC after error@2112184578)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.7.175.254:39253,127.7.175.252:42903,127.7.175.253:43719) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.253:43719(127.7.175.253:43719): Connection refused: /127.7.175.253:43719,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.175.254:39253(127.7.175.254:39253): Connection refused: /127.7.175.254:39253
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.7.175.254:39253
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)