16:19:31.787 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
16:19:31.800 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskVGVFlJ/test-tmp
16:19:31.803 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
16:19:31.806 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskVGVFlJ/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
16:19:32.047 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
16:19:32.218 [DEBUG - main] (MiniKuduCluster.java:195) Response:
16:19:32.219 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
16:19:32.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:32.223326 5368 mini_oidc.cc:152] Starting JWKS server
16:19:32.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:32.227062 5368 webserver.cc:460] Webserver started at http://127.0.0.1:34851/ using document root <none> and password file <none>
16:19:32.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:32.227674 5368 mini_oidc.cc:189] Starting OIDC Discovery server
16:19:32.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:32.229635 5368 webserver.cc:460] Webserver started at http://127.0.0.1:45911/ using document root <none> and password file <none>
16:19:32.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-11T16:19:32Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
16:19:32.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-11T16:19:32Z Disabled control of system clock
16:19:32.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:32.283259 5368 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVGVFlJ/build/tsan/bin/kudu
16:19:32.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskVGVFlJ/build/tsan/bin/kudu
16:19:32.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/master-0/wal
16:19:32.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/master-0/data
16:19:32.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:19:32.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:19:32.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:19:32.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:19:32.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:19:32.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:19:32.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/master-0/logs
16:19:32.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/master-0/data/info.pb
16:19:32.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:19:32.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:19:32.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:19:32.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:19:32.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:19:32.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:19:32.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:19:32.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:19:32.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:19:32.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:19:32.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:19:32.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:19:32.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.5.62.62:36133
16:19:32.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.5.62.62
16:19:32.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:19:32.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.5.62.20:35173
16:19:32.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:19:32.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:19:32.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:19:32.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:19:32.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.5.62.62:36133,127.5.62.61:41997,127.5.62.60:33963
16:19:32.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:19:32.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34851/jwks.json with env {}
16:19:32.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:32.601667 5382 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:19:32.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:32.602260 5382 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:19:32.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:32.602928 5382 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:19:32.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:32.634727 5382 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:19:32.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:32.635025 5382 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:19:32.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:32.635289 5382 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:19:32.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:32.635478 5382 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:19:32.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:32.635803 5382 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:19:32.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:32.636024 5382 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34851/jwks.json
16:19:32.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:32.675730 5382 master_runner.cc:386] Master server non-default flags:
16:19:32.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:19:32.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.5.62.20:35173
16:19:32.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:19:32.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:19:32.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/master-0/data
16:19:32.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/master-0/wal
16:19:32.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:19:32.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.5.62.62:36133,127.5.62.61:41997,127.5.62.60:33963
16:19:32.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:19:32.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:19:32.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:19:32.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.5.62.62:36133
16:19:32.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:19:32.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:19:32.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:19:32.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34851/jwks.json
16:19:32.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:19:32.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:19:32.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/master-0/data/info.pb
16:19:32.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.5.62.62
16:19:32.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:19:32.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:19:32.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:19:32.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:19:32.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:19:32.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:19:32.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/master-0/logs
16:19:32.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:19:32.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:19:32.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:19:32.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:19:32.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
16:19:32.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
16:19:32.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:19:32.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 15:12:52 UTC on ccycloud.quasar-tzrfpd.root.comops.site
16:19:32.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4580143
16:19:32.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:19:32.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:32.677448 5382 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:19:32.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:32.679812 5382 file_cache.cc:493] Constructed file cache file cache with capacity 419430
16:19:32.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:32.701043 5392 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:33.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:32.701088 5390 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:33.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:32.701135 5388 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:33.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:33.890038 5391 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
16:19:33.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:33.890170 5382 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:19:33.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:33.894263 5382 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:19:33.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:33.896867 5382 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:19:33.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:33.898202 5382 hybrid_clock.cc:648] HybridClock initialized: now 1786465173898141 us; error 110 us; skew 500 ppm
16:19:33.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:33.898980 5382 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:19:33.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:33.905658 5382 webserver.cc:460] Webserver started at http://127.5.62.62:44321/ using document root <none> and password file <none>
16:19:33.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:33.906579 5382 fs_manager.cc:359] Metadata directory not provided
16:19:33.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:33.906762 5382 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:19:33.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:33.907150 5382 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
16:19:33.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:33.911966 5382 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/master-0/data/instance:
16:19:33.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7c9e99a26a2e42829ec7114980e6dc63"
16:19:33.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 16:19:33 on dist-test-slave-cbbz"
16:19:33.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:33.913201 5382 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/master-0/wal/instance:
16:19:33.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7c9e99a26a2e42829ec7114980e6dc63"
16:19:33.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 16:19:33 on dist-test-slave-cbbz"
16:19:33.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:33.920166 5382 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.007s sys 0.000s
16:19:33.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:33.925661 5400 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:19:33.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:33.926789 5382 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.003s
16:19:33.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:33.927168 5382 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/master-0/data,/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/master-0/wal
16:19:33.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7c9e99a26a2e42829ec7114980e6dc63"
16:19:33.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 16:19:33 on dist-test-slave-cbbz"
16:19:33.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:33.927501 5382 fs_report.cc:383] FS layout report
16:19:33.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:19:33.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/master-0/wal
16:19:33.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/master-0/wal
16:19:33.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/master-0/data/data
16:19:33.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:19:33.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:19:33.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:19:33.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:19:33.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:19:33.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:19:33.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:19:33.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:19:33.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:19:33.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:19:33.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:19:33.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:33.971614 5382 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
16:19:33.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:33.973156 5382 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:19:33.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:33.973569 5382 kserver.cc:163] Server-wide thread pool size limit: 3276
16:19:34.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:34.047366 5382 rpc_server.cc:307] RPC server started. Bound to: 127.5.62.62:36133
16:19:34.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:34.047497 5457 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.5.62.62:36133 every 8 connection(s)
16:19:34.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:34.050222 5382 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/master-0/data/info.pb
16:19:34.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:34.055482 5458 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
16:19:34.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:34.057015 5368 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVGVFlJ/build/tsan/bin/kudu as pid 5382
16:19:34.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:34.057421 5368 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/master-0/wal/instance
16:19:34.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:34.061707 5368 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVGVFlJ/build/tsan/bin/kudu
16:19:34.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskVGVFlJ/build/tsan/bin/kudu
16:19:34.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/master-1/wal
16:19:34.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/master-1/data
16:19:34.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:19:34.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:19:34.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:19:34.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:19:34.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:19:34.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:19:34.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/master-1/logs
16:19:34.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/master-1/data/info.pb
16:19:34.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:19:34.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:19:34.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:19:34.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:19:34.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:19:34.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:19:34.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:19:34.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:19:34.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:19:34.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:19:34.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:19:34.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:19:34.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.5.62.61:41997
16:19:34.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.5.62.61
16:19:34.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:19:34.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.5.62.20:35173
16:19:34.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:19:34.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:19:34.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:19:34.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:19:34.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.5.62.62:36133,127.5.62.61:41997,127.5.62.60:33963
16:19:34.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:19:34.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34851/jwks.json with env {}
16:19:34.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:34.069772 5458 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.5.62.62" port: 36133 } has no permanent_uuid. Determining permanent_uuid...
16:19:34.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:34.099936 5458 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.5.62.61" port: 41997 } has no permanent_uuid. Determining permanent_uuid...
16:19:34.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:34.102293 5403 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.5.62.61:41997: connect: Connection refused (error 111)
16:19:34.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:34.106762 5458 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.62.61:41997: Network error: Client connection negotiation failed: client connection to 127.5.62.61:41997: connect: Connection refused (error 111)
16:19:34.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:34.156327 5458 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.62.61" port: 41997 } attempt: 1
16:19:34.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:34.160356 5458 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.62.61:41997: Network error: Client connection negotiation failed: client connection to 127.5.62.61:41997: connect: Connection refused (error 111)
16:19:34.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:34.228843 5458 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.62.61" port: 41997 } attempt: 2
16:19:34.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:34.233013 5458 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.62.61:41997: Network error: Client connection negotiation failed: client connection to 127.5.62.61:41997: connect: Connection refused (error 111)
16:19:34.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:34.324426 5458 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.62.61" port: 41997 } attempt: 3
16:19:34.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:34.328744 5458 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.62.61:41997: Network error: Client connection negotiation failed: client connection to 127.5.62.61:41997: connect: Connection refused (error 111)
16:19:34.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:34.395648 5460 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:19:34.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:34.396323 5460 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:19:34.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:34.396961 5460 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:19:34.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:34.429419 5460 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:19:34.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:34.429703 5460 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:19:34.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:34.429895 5460 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:19:34.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:34.430071 5460 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:19:34.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:34.430259 5460 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:19:34.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:34.430490 5460 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34851/jwks.json
16:19:34.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:34.466694 5460 master_runner.cc:386] Master server non-default flags:
16:19:34.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:19:34.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.5.62.20:35173
16:19:34.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:19:34.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:19:34.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/master-1/data
16:19:34.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/master-1/wal
16:19:34.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:19:34.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.5.62.62:36133,127.5.62.61:41997,127.5.62.60:33963
16:19:34.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:19:34.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:19:34.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:19:34.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.5.62.61:41997
16:19:34.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:19:34.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:19:34.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:19:34.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34851/jwks.json
16:19:34.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:19:34.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:19:34.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/master-1/data/info.pb
16:19:34.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.5.62.61
16:19:34.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:19:34.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:19:34.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:19:34.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:19:34.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:19:34.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:19:34.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/master-1/logs
16:19:34.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:19:34.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:19:34.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:19:34.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:19:34.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
16:19:34.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
16:19:34.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:19:34.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 15:12:52 UTC on ccycloud.quasar-tzrfpd.root.comops.site
16:19:34.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4580143
16:19:34.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:19:34.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:34.467954 5460 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:19:34.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:34.470423 5460 file_cache.cc:493] Constructed file cache file cache with capacity 419430
16:19:34.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:34.472121 5458 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.62.61" port: 41997 } attempt: 4
16:19:34.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:34.475976 5458 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.62.61:41997: Network error: Client connection negotiation failed: client connection to 127.5.62.61:41997: connect: Connection refused (error 111)
16:19:34.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:34.492266 5473 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:34.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:34.775501 5458 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.62.61" port: 41997 } attempt: 5
16:19:34.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:34.779855 5458 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.62.61:41997: Network error: Client connection negotiation failed: client connection to 127.5.62.61:41997: connect: Connection refused (error 111)
16:19:35.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:35.327495 5458 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.62.61" port: 41997 } attempt: 6
16:19:35.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:35.338353 5458 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.62.61:41997: Network error: Client connection negotiation failed: client connection to 127.5.62.61:41997: connect: Connection refused (error 111)
16:19:35.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:34.492393 5472 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:35.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:34.493044 5475 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:35.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:34.493886 5460 server_base.cc:1034] running on GCE node
16:19:35.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:35.694471 5460 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:19:35.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:35.697916 5460 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:19:35.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:35.699368 5460 hybrid_clock.cc:648] HybridClock initialized: now 1786465175699320 us; error 68 us; skew 500 ppm
16:19:35.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:35.700276 5460 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:19:35.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:35.706750 5460 webserver.cc:460] Webserver started at http://127.5.62.61:46613/ using document root <none> and password file <none>
16:19:35.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:35.707789 5460 fs_manager.cc:359] Metadata directory not provided
16:19:35.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:35.707984 5460 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:19:35.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:35.708464 5460 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
16:19:35.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:35.713344 5460 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/master-1/data/instance:
16:19:35.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "82dc816b1d3142de823d77d0e689dc45"
16:19:35.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 16:19:35 on dist-test-slave-cbbz"
16:19:35.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:35.714438 5460 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/master-1/wal/instance:
16:19:35.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "82dc816b1d3142de823d77d0e689dc45"
16:19:35.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 16:19:35 on dist-test-slave-cbbz"
16:19:35.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:35.721587 5460 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.008s sys 0.000s
16:19:35.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:35.727198 5486 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:19:35.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:35.728384 5460 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
16:19:35.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:35.728663 5460 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/master-1/data,/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/master-1/wal
16:19:35.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "82dc816b1d3142de823d77d0e689dc45"
16:19:35.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 16:19:35 on dist-test-slave-cbbz"
16:19:35.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:35.728968 5460 fs_report.cc:383] FS layout report
16:19:35.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:19:35.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/master-1/wal
16:19:35.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/master-1/wal
16:19:35.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/master-1/data/data
16:19:35.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:19:35.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:19:35.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:19:35.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:19:35.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:19:35.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:19:35.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:19:35.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:19:35.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:19:35.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:19:35.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:19:35.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:35.776612 5460 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
16:19:35.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:35.778088 5460 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:19:35.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:35.778522 5460 kserver.cc:163] Server-wide thread pool size limit: 3276
16:19:35.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:35.852234 5460 rpc_server.cc:307] RPC server started. Bound to: 127.5.62.61:41997
16:19:35.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:35.852335 5540 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.5.62.61:41997 every 8 connection(s)
16:19:35.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:35.854892 5460 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/master-1/data/info.pb
16:19:35.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:35.856293 5368 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVGVFlJ/build/tsan/bin/kudu as pid 5460
16:19:35.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:35.856868 5368 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/master-1/wal/instance
16:19:35.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:35.859752 5368 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVGVFlJ/build/tsan/bin/kudu
16:19:35.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskVGVFlJ/build/tsan/bin/kudu
16:19:35.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/master-2/wal
16:19:35.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/master-2/data
16:19:35.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:19:35.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:19:35.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:19:35.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:19:35.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:19:35.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:19:35.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/master-2/logs
16:19:35.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/master-2/data/info.pb
16:19:35.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:19:35.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:19:35.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:19:35.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:19:35.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:19:35.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:19:35.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:19:35.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:19:35.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:19:35.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:19:35.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:19:35.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:19:35.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.5.62.60:33963
16:19:35.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.5.62.60
16:19:35.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:19:35.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.5.62.20:35173
16:19:35.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:19:35.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:19:35.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:19:35.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:19:35.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.5.62.62:36133,127.5.62.61:41997,127.5.62.60:33963
16:19:35.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:19:35.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34851/jwks.json with env {}
16:19:35.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:35.861590 5542 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
16:19:35.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:35.873309 5542 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.5.62.62" port: 36133 } has no permanent_uuid. Determining permanent_uuid...
16:19:35.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:35.897324 5542 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.5.62.61" port: 41997 } has no permanent_uuid. Determining permanent_uuid...
16:19:35.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:35.914314 5542 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.5.62.60" port: 33963 } has no permanent_uuid. Determining permanent_uuid...
16:19:35.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:35.916940 5487 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.5.62.60:33963: connect: Connection refused (error 111)
16:19:35.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:35.921264 5542 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.62.60:33963: Network error: Client connection negotiation failed: client connection to 127.5.62.60:33963: connect: Connection refused (error 111)
16:19:35.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:35.970896 5542 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.62.60" port: 33963 } attempt: 1
16:19:35.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:35.974891 5542 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.62.60:33963: Network error: Client connection negotiation failed: client connection to 127.5.62.60:33963: connect: Connection refused (error 111)
16:19:36.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:36.043282 5542 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.62.60" port: 33963 } attempt: 2
16:19:36.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:36.047137 5542 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.62.60:33963: Network error: Client connection negotiation failed: client connection to 127.5.62.60:33963: connect: Connection refused (error 111)
16:19:36.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:36.138592 5542 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.62.60" port: 33963 } attempt: 3
16:19:36.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:36.142939 5542 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.62.60:33963: Network error: Client connection negotiation failed: client connection to 127.5.62.60:33963: connect: Connection refused (error 111)
16:19:36.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:36.190686 5544 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:19:36.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:36.191427 5544 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:19:36.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:36.192116 5544 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:19:36.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:36.223239 5544 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:19:36.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:36.223694 5544 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:19:36.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:36.224035 5544 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:19:36.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:36.224421 5544 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:19:36.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:36.224759 5544 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:19:36.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:36.225019 5544 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34851/jwks.json
16:19:36.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:36.261986 5544 master_runner.cc:386] Master server non-default flags:
16:19:36.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:19:36.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.5.62.20:35173
16:19:36.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:19:36.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:19:36.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/master-2/data
16:19:36.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/master-2/wal
16:19:36.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:19:36.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.5.62.62:36133,127.5.62.61:41997,127.5.62.60:33963
16:19:36.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:19:36.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:19:36.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:19:36.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.5.62.60:33963
16:19:36.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:19:36.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:19:36.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:19:36.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34851/jwks.json
16:19:36.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:19:36.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:19:36.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/master-2/data/info.pb
16:19:36.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.5.62.60
16:19:36.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:19:36.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:19:36.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:19:36.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:19:36.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:19:36.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:19:36.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/master-2/logs
16:19:36.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:19:36.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:19:36.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:19:36.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:19:36.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
16:19:36.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
16:19:36.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:19:36.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 15:12:52 UTC on ccycloud.quasar-tzrfpd.root.comops.site
16:19:36.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4580143
16:19:36.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:19:36.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:36.263556 5544 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:19:36.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:36.265821 5544 file_cache.cc:493] Constructed file cache file cache with capacity 419430
16:19:36.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:36.285045 5555 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:36.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:36.286392 5542 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.62.60" port: 33963 } attempt: 4
16:19:36.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:36.290481 5542 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.62.60:33963: Network error: Client connection negotiation failed: client connection to 127.5.62.60:33963: connect: Connection refused (error 111)
16:19:36.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:36.398973 5458 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.62.61" port: 41997 } attempt: 7
16:19:36.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:36.417137 5458 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.5.62.60" port: 33963 } has no permanent_uuid. Determining permanent_uuid...
16:19:36.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:36.423755 5458 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.62.60:33963: Network error: Client connection negotiation failed: client connection to 127.5.62.60:33963: connect: Connection refused (error 111)
16:19:36.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:36.482158 5458 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.62.60" port: 33963 } attempt: 1
16:19:36.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:36.486747 5458 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.62.60:33963: Network error: Client connection negotiation failed: client connection to 127.5.62.60:33963: connect: Connection refused (error 111)
16:19:36.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:36.568203 5458 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.62.60" port: 33963 } attempt: 2
16:19:36.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:36.572943 5458 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.62.60:33963: Network error: Client connection negotiation failed: client connection to 127.5.62.60:33963: connect: Connection refused (error 111)
16:19:36.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:36.589957 5542 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.62.60" port: 33963 } attempt: 5
16:19:36.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:36.594743 5542 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.62.60:33963: Network error: Client connection negotiation failed: client connection to 127.5.62.60:33963: connect: Connection refused (error 111)
16:19:36.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:36.658435 5458 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.62.60" port: 33963 } attempt: 3
16:19:36.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:36.662921 5458 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.62.60:33963: Network error: Client connection negotiation failed: client connection to 127.5.62.60:33963: connect: Connection refused (error 111)
16:19:36.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:36.803392 5458 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.62.60" port: 33963 } attempt: 4
16:19:36.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:36.808060 5458 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.62.60:33963: Network error: Client connection negotiation failed: client connection to 127.5.62.60:33963: connect: Connection refused (error 111)
16:19:37.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.091493 5458 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.62.60" port: 33963 } attempt: 5
16:19:37.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:37.095930 5458 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.62.60:33963: Network error: Client connection negotiation failed: client connection to 127.5.62.60:33963: connect: Connection refused (error 111)
16:19:37.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.142122 5542 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.62.60" port: 33963 } attempt: 6
16:19:37.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:37.149359 5542 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.62.60:33963: Network error: Client connection negotiation failed: client connection to 127.5.62.60:33963: connect: Connection refused (error 111)
16:19:37.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:36.285102 5559 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:37.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:36.286757 5544 server_base.cc:1034] running on GCE node
16:19:37.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:36.286185 5556 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:37.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.487190 5544 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:19:37.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.489990 5544 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:19:37.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.491393 5544 hybrid_clock.cc:648] HybridClock initialized: now 1786465177491364 us; error 39 us; skew 500 ppm
16:19:37.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.492411 5544 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:19:37.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.503911 5544 webserver.cc:460] Webserver started at http://127.5.62.60:37159/ using document root <none> and password file <none>
16:19:37.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.504992 5544 fs_manager.cc:359] Metadata directory not provided
16:19:37.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.505255 5544 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:19:37.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.505870 5544 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
16:19:37.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.511366 5544 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/master-2/data/instance:
16:19:37.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "65d34a1d9c7242368eae1c56211a6b90"
16:19:37.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 16:19:37 on dist-test-slave-cbbz"
16:19:37.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.512815 5544 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/master-2/wal/instance:
16:19:37.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "65d34a1d9c7242368eae1c56211a6b90"
16:19:37.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 16:19:37 on dist-test-slave-cbbz"
16:19:37.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.520798 5544 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.007s sys 0.000s
16:19:37.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.526639 5571 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:19:37.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.527882 5544 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
16:19:37.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.528311 5544 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/master-2/data,/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/master-2/wal
16:19:37.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "65d34a1d9c7242368eae1c56211a6b90"
16:19:37.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 16:19:37 on dist-test-slave-cbbz"
16:19:37.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.528746 5544 fs_report.cc:383] FS layout report
16:19:37.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:19:37.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/master-2/wal
16:19:37.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/master-2/wal
16:19:37.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/master-2/data/data
16:19:37.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:19:37.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:19:37.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:19:37.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:19:37.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:19:37.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:19:37.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:19:37.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:19:37.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:19:37.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:19:37.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:19:37.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.567030 5544 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
16:19:37.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.568605 5544 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:19:37.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.569029 5544 kserver.cc:163] Server-wide thread pool size limit: 3276
16:19:37.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.642730 5544 rpc_server.cc:307] RPC server started. Bound to: 127.5.62.60:33963
16:19:37.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.642838 5629 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.5.62.60:33963 every 8 connection(s)
16:19:37.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.645442 5544 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/master-2/data/info.pb
16:19:37.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.648557 5458 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.62.60" port: 33963 } attempt: 6
16:19:37.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.652673 5630 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
16:19:37.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.655741 5368 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVGVFlJ/build/tsan/bin/kudu as pid 5544
16:19:37.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.656219 5368 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/master-2/wal/instance
16:19:37.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.672840 5630 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.5.62.62" port: 36133 } has no permanent_uuid. Determining permanent_uuid...
16:19:37.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.711493 5630 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.5.62.61" port: 41997 } has no permanent_uuid. Determining permanent_uuid...
16:19:37.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.719184 5458 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7c9e99a26a2e42829ec7114980e6dc63: Bootstrap starting.
16:19:37.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.728479 5458 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 7c9e99a26a2e42829ec7114980e6dc63: Neither blocks nor log segments found. Creating new log.
16:19:37.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.730079 5630 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.5.62.60" port: 33963 } has no permanent_uuid. Determining permanent_uuid...
16:19:37.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.730815 5458 log.cc:824] T 00000000000000000000000000000000 P 7c9e99a26a2e42829ec7114980e6dc63: Log is configured to *not* fsync() on all Append() calls
16:19:37.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.739106 5458 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7c9e99a26a2e42829ec7114980e6dc63: No bootstrap required, opened a new log
16:19:37.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.753326 5630 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 65d34a1d9c7242368eae1c56211a6b90: Bootstrap starting.
16:19:37.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.760996 5630 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 65d34a1d9c7242368eae1c56211a6b90: Neither blocks nor log segments found. Creating new log.
16:19:37.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.763428 5630 log.cc:824] T 00000000000000000000000000000000 P 65d34a1d9c7242368eae1c56211a6b90: Log is configured to *not* fsync() on all Append() calls
16:19:37.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.768347 5458 raft_consensus.cc:348] T 00000000000000000000000000000000 P 7c9e99a26a2e42829ec7114980e6dc63 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c9e99a26a2e42829ec7114980e6dc63" member_type: VOTER last_known_addr { host: "127.5.62.62" port: 36133 } } peers { permanent_uuid: "82dc816b1d3142de823d77d0e689dc45" member_type: VOTER last_known_addr { host: "127.5.62.61" port: 41997 } } peers { permanent_uuid: "65d34a1d9c7242368eae1c56211a6b90" member_type: VOTER last_known_addr { host: "127.5.62.60" port: 33963 } }
16:19:37.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.769049 5458 raft_consensus.cc:374] T 00000000000000000000000000000000 P 7c9e99a26a2e42829ec7114980e6dc63 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
16:19:37.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.769196 5630 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 65d34a1d9c7242368eae1c56211a6b90: No bootstrap required, opened a new log
16:19:37.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.769346 5458 raft_consensus.cc:729] T 00000000000000000000000000000000 P 7c9e99a26a2e42829ec7114980e6dc63 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7c9e99a26a2e42829ec7114980e6dc63, State: Initialized, Role: FOLLOWER
16:19:37.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.770529 5458 consensus_queue.cc:260] T 00000000000000000000000000000000 P 7c9e99a26a2e42829ec7114980e6dc63 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c9e99a26a2e42829ec7114980e6dc63" member_type: VOTER last_known_addr { host: "127.5.62.62" port: 36133 } } peers { permanent_uuid: "82dc816b1d3142de823d77d0e689dc45" member_type: VOTER last_known_addr { host: "127.5.62.61" port: 41997 } } peers { permanent_uuid: "65d34a1d9c7242368eae1c56211a6b90" member_type: VOTER last_known_addr { host: "127.5.62.60" port: 33963 } }
16:19:37.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.774837 5645 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7c9e99a26a2e42829ec7114980e6dc63 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c9e99a26a2e42829ec7114980e6dc63" member_type: VOTER last_known_addr { host: "127.5.62.62" port: 36133 } } peers { permanent_uuid: "82dc816b1d3142de823d77d0e689dc45" member_type: VOTER last_known_addr { host: "127.5.62.61" port: 41997 } } peers { permanent_uuid: "65d34a1d9c7242368eae1c56211a6b90" member_type: VOTER last_known_addr { host: "127.5.62.60" port: 33963 } } }
16:19:37.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.775714 5645 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7c9e99a26a2e42829ec7114980e6dc63 [sys.catalog]: This master's current role is: FOLLOWER
16:19:37.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.776793 5458 sys_catalog.cc:564] T 00000000000000000000000000000000 P 7c9e99a26a2e42829ec7114980e6dc63 [sys.catalog]: configured and running, proceeding with master startup.
16:19:37.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.799158 5630 raft_consensus.cc:348] T 00000000000000000000000000000000 P 65d34a1d9c7242368eae1c56211a6b90 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c9e99a26a2e42829ec7114980e6dc63" member_type: VOTER last_known_addr { host: "127.5.62.62" port: 36133 } } peers { permanent_uuid: "82dc816b1d3142de823d77d0e689dc45" member_type: VOTER last_known_addr { host: "127.5.62.61" port: 41997 } } peers { permanent_uuid: "65d34a1d9c7242368eae1c56211a6b90" member_type: VOTER last_known_addr { host: "127.5.62.60" port: 33963 } }
16:19:37.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.800206 5630 raft_consensus.cc:374] T 00000000000000000000000000000000 P 65d34a1d9c7242368eae1c56211a6b90 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
16:19:37.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.800505 5630 raft_consensus.cc:729] T 00000000000000000000000000000000 P 65d34a1d9c7242368eae1c56211a6b90 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 65d34a1d9c7242368eae1c56211a6b90, State: Initialized, Role: FOLLOWER
16:19:37.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.801462 5630 consensus_queue.cc:260] T 00000000000000000000000000000000 P 65d34a1d9c7242368eae1c56211a6b90 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c9e99a26a2e42829ec7114980e6dc63" member_type: VOTER last_known_addr { host: "127.5.62.62" port: 36133 } } peers { permanent_uuid: "82dc816b1d3142de823d77d0e689dc45" member_type: VOTER last_known_addr { host: "127.5.62.61" port: 41997 } } peers { permanent_uuid: "65d34a1d9c7242368eae1c56211a6b90" member_type: VOTER last_known_addr { host: "127.5.62.60" port: 33963 } }
16:19:37.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.804898 5657 sys_catalog.cc:455] T 00000000000000000000000000000000 P 65d34a1d9c7242368eae1c56211a6b90 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c9e99a26a2e42829ec7114980e6dc63" member_type: VOTER last_known_addr { host: "127.5.62.62" port: 36133 } } peers { permanent_uuid: "82dc816b1d3142de823d77d0e689dc45" member_type: VOTER last_known_addr { host: "127.5.62.61" port: 41997 } } peers { permanent_uuid: "65d34a1d9c7242368eae1c56211a6b90" member_type: VOTER last_known_addr { host: "127.5.62.60" port: 33963 } } }
16:19:37.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.805809 5657 sys_catalog.cc:458] T 00000000000000000000000000000000 P 65d34a1d9c7242368eae1c56211a6b90 [sys.catalog]: This master's current role is: FOLLOWER
16:19:37.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.806375 5630 sys_catalog.cc:564] T 00000000000000000000000000000000 P 65d34a1d9c7242368eae1c56211a6b90 [sys.catalog]: configured and running, proceeding with master startup.
16:19:37.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:37.824268 5577 tablet.cc:2369] T 00000000000000000000000000000000 P 65d34a1d9c7242368eae1c56211a6b90: Can't schedule compaction. Clean time has not been advanced past its initial value.
16:19:37.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:37.839614 5656 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7c9e99a26a2e42829ec7114980e6dc63: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:19:37.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:37.840376 5656 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:19:37.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:37.857723 5669 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 65d34a1d9c7242368eae1c56211a6b90: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:19:37.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:37.858068 5669 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:19:37.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.809057 5382 master_runner.cc:186] Error getting master registration for 127.5.62.61:41997: OK, instance_id { permanent_uuid: "82dc816b1d3142de823d77d0e689dc45" instance_seqno: 1786465175831275 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
16:19:37.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.892367 5382 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:19:37.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.906805 5645 raft_consensus.cc:482] T 00000000000000000000000000000000 P 7c9e99a26a2e42829ec7114980e6dc63 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
16:19:37.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.907368 5645 raft_consensus.cc:504] T 00000000000000000000000000000000 P 7c9e99a26a2e42829ec7114980e6dc63 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c9e99a26a2e42829ec7114980e6dc63" member_type: VOTER last_known_addr { host: "127.5.62.62" port: 36133 } } peers { permanent_uuid: "82dc816b1d3142de823d77d0e689dc45" member_type: VOTER last_known_addr { host: "127.5.62.61" port: 41997 } } peers { permanent_uuid: "65d34a1d9c7242368eae1c56211a6b90" member_type: VOTER last_known_addr { host: "127.5.62.60" port: 33963 } }
16:19:37.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.854511 5544 master_runner.cc:186] Error getting master registration for 127.5.62.61:41997: OK, instance_id { permanent_uuid: "82dc816b1d3142de823d77d0e689dc45" instance_seqno: 1786465175831275 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
16:19:37.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.909713 5544 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:19:37.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.913018 5645 leader_election.cc:288] T 00000000000000000000000000000000 P 7c9e99a26a2e42829ec7114980e6dc63 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82dc816b1d3142de823d77d0e689dc45 (127.5.62.61:41997), 65d34a1d9c7242368eae1c56211a6b90 (127.5.62.60:33963)
16:19:37.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.913452 5600 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7c9e99a26a2e42829ec7114980e6dc63" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "65d34a1d9c7242368eae1c56211a6b90" is_pre_election: true
16:19:37.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.914517 5600 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 65d34a1d9c7242368eae1c56211a6b90 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7c9e99a26a2e42829ec7114980e6dc63 in term 0.
16:19:37.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.916488 5401 leader_election.cc:302] T 00000000000000000000000000000000 P 7c9e99a26a2e42829ec7114980e6dc63 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 65d34a1d9c7242368eae1c56211a6b90, 7c9e99a26a2e42829ec7114980e6dc63; no voters:
16:19:37.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.918601 5645 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 7c9e99a26a2e42829ec7114980e6dc63 [term 0 FOLLOWER]: Leader pre-election won for term 1
16:19:37.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.919440 5645 raft_consensus.cc:482] T 00000000000000000000000000000000 P 7c9e99a26a2e42829ec7114980e6dc63 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
16:19:37.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.920254 5645 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 7c9e99a26a2e42829ec7114980e6dc63 [term 0 FOLLOWER]: Advancing to term 1
16:19:37.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.932641 5645 raft_consensus.cc:504] T 00000000000000000000000000000000 P 7c9e99a26a2e42829ec7114980e6dc63 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c9e99a26a2e42829ec7114980e6dc63" member_type: VOTER last_known_addr { host: "127.5.62.62" port: 36133 } } peers { permanent_uuid: "82dc816b1d3142de823d77d0e689dc45" member_type: VOTER last_known_addr { host: "127.5.62.61" port: 41997 } } peers { permanent_uuid: "65d34a1d9c7242368eae1c56211a6b90" member_type: VOTER last_known_addr { host: "127.5.62.60" port: 33963 } }
16:19:37.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.937031 5645 leader_election.cc:288] T 00000000000000000000000000000000 P 7c9e99a26a2e42829ec7114980e6dc63 [CANDIDATE]: Term 1 election: Requested vote from peers 82dc816b1d3142de823d77d0e689dc45 (127.5.62.61:41997), 65d34a1d9c7242368eae1c56211a6b90 (127.5.62.60:33963)
16:19:37.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.939738 5600 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7c9e99a26a2e42829ec7114980e6dc63" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "65d34a1d9c7242368eae1c56211a6b90"
16:19:37.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.940337 5600 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 65d34a1d9c7242368eae1c56211a6b90 [term 0 FOLLOWER]: Advancing to term 1
16:19:37.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.947207 5600 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 65d34a1d9c7242368eae1c56211a6b90 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7c9e99a26a2e42829ec7114980e6dc63 in term 1.
16:19:37.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.959826 5401 leader_election.cc:302] T 00000000000000000000000000000000 P 7c9e99a26a2e42829ec7114980e6dc63 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 65d34a1d9c7242368eae1c56211a6b90, 7c9e99a26a2e42829ec7114980e6dc63; no voters:
16:19:37.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.960896 5645 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 7c9e99a26a2e42829ec7114980e6dc63 [term 1 FOLLOWER]: Leader election won for term 1
16:19:37.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.967293 5645 raft_consensus.cc:686] T 00000000000000000000000000000000 P 7c9e99a26a2e42829ec7114980e6dc63 [term 1 LEADER]: Becoming Leader. State: Replica: 7c9e99a26a2e42829ec7114980e6dc63, State: Running, Role: LEADER
16:19:37.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.969592 5645 consensus_queue.cc:237] T 00000000000000000000000000000000 P 7c9e99a26a2e42829ec7114980e6dc63 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c9e99a26a2e42829ec7114980e6dc63" member_type: VOTER last_known_addr { host: "127.5.62.62" port: 36133 } } peers { permanent_uuid: "82dc816b1d3142de823d77d0e689dc45" member_type: VOTER last_known_addr { host: "127.5.62.61" port: 41997 } } peers { permanent_uuid: "65d34a1d9c7242368eae1c56211a6b90" member_type: VOTER last_known_addr { host: "127.5.62.60" port: 33963 } }
16:19:37.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:37.982720 5407 tablet.cc:2369] T 00000000000000000000000000000000 P 7c9e99a26a2e42829ec7114980e6dc63: Can't schedule compaction. Clean time has not been advanced past its initial value.
16:19:37.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.997479 5671 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7c9e99a26a2e42829ec7114980e6dc63 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 7c9e99a26a2e42829ec7114980e6dc63. Latest consensus state: current_term: 1 leader_uuid: "7c9e99a26a2e42829ec7114980e6dc63" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c9e99a26a2e42829ec7114980e6dc63" member_type: VOTER last_known_addr { host: "127.5.62.62" port: 36133 } } peers { permanent_uuid: "82dc816b1d3142de823d77d0e689dc45" member_type: VOTER last_known_addr { host: "127.5.62.61" port: 41997 } } peers { permanent_uuid: "65d34a1d9c7242368eae1c56211a6b90" member_type: VOTER last_known_addr { host: "127.5.62.60" port: 33963 } } }
16:19:37.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:37.998353 5671 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7c9e99a26a2e42829ec7114980e6dc63 [sys.catalog]: This master's current role is: LEADER
16:19:38.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:38.004523 5673 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
16:19:38.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:38.014423 5673 catalog_manager.cc:1484] Initializing Kudu cluster ID...
16:19:38.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:38.049799 5600 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 65d34a1d9c7242368eae1c56211a6b90 [term 1 FOLLOWER]: Refusing update from remote peer 7c9e99a26a2e42829ec7114980e6dc63: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
16:19:38.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:38.059420 5671 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 7c9e99a26a2e42829ec7114980e6dc63 [LEADER]: Connected to new peer: Peer: permanent_uuid: "65d34a1d9c7242368eae1c56211a6b90" member_type: VOTER last_known_addr { host: "127.5.62.60" port: 33963 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
16:19:38.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:38.099622 5657 sys_catalog.cc:455] T 00000000000000000000000000000000 P 65d34a1d9c7242368eae1c56211a6b90 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 7c9e99a26a2e42829ec7114980e6dc63. Latest consensus state: current_term: 1 leader_uuid: "7c9e99a26a2e42829ec7114980e6dc63" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c9e99a26a2e42829ec7114980e6dc63" member_type: VOTER last_known_addr { host: "127.5.62.62" port: 36133 } } peers { permanent_uuid: "82dc816b1d3142de823d77d0e689dc45" member_type: VOTER last_known_addr { host: "127.5.62.61" port: 41997 } } peers { permanent_uuid: "65d34a1d9c7242368eae1c56211a6b90" member_type: VOTER last_known_addr { host: "127.5.62.60" port: 33963 } } }
16:19:38.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:38.100416 5657 sys_catalog.cc:458] T 00000000000000000000000000000000 P 65d34a1d9c7242368eae1c56211a6b90 [sys.catalog]: This master's current role is: FOLLOWER
16:19:38.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:38.109575 5671 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7c9e99a26a2e42829ec7114980e6dc63 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "7c9e99a26a2e42829ec7114980e6dc63" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c9e99a26a2e42829ec7114980e6dc63" member_type: VOTER last_known_addr { host: "127.5.62.62" port: 36133 } } peers { permanent_uuid: "82dc816b1d3142de823d77d0e689dc45" member_type: VOTER last_known_addr { host: "127.5.62.61" port: 41997 } } peers { permanent_uuid: "65d34a1d9c7242368eae1c56211a6b90" member_type: VOTER last_known_addr { host: "127.5.62.60" port: 33963 } } }
16:19:38.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:38.110354 5671 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7c9e99a26a2e42829ec7114980e6dc63 [sys.catalog]: This master's current role is: LEADER
16:19:38.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:38.119014 5657 sys_catalog.cc:455] T 00000000000000000000000000000000 P 65d34a1d9c7242368eae1c56211a6b90 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "7c9e99a26a2e42829ec7114980e6dc63" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c9e99a26a2e42829ec7114980e6dc63" member_type: VOTER last_known_addr { host: "127.5.62.62" port: 36133 } } peers { permanent_uuid: "82dc816b1d3142de823d77d0e689dc45" member_type: VOTER last_known_addr { host: "127.5.62.61" port: 41997 } } peers { permanent_uuid: "65d34a1d9c7242368eae1c56211a6b90" member_type: VOTER last_known_addr { host: "127.5.62.60" port: 33963 } } }
16:19:38.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:38.119778 5657 sys_catalog.cc:458] T 00000000000000000000000000000000 P 65d34a1d9c7242368eae1c56211a6b90 [sys.catalog]: This master's current role is: FOLLOWER
16:19:38.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:38.130178 5673 catalog_manager.cc:1347] Generated new cluster ID: c17879e963c3420d9cb1eb2bcb7eff67
16:19:38.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:38.130498 5673 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
16:19:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:38.199568 5673 catalog_manager.cc:1370] Generated new certificate authority record
16:19:38.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:38.204768 5673 catalog_manager.cc:1504] Loading token signing keys...
16:19:38.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:38.209983 5542 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.62.60" port: 33963 } attempt: 7
16:19:38.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:38.282683 5542 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 82dc816b1d3142de823d77d0e689dc45: Bootstrap starting.
16:19:38.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:38.335671 5673 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 7c9e99a26a2e42829ec7114980e6dc63: Generated new TSK 0
16:19:38.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:38.346413 5673 catalog_manager.cc:1514] Initializing in-progress tserver states...
16:19:38.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:38.378479 5542 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 82dc816b1d3142de823d77d0e689dc45: Neither blocks nor log segments found. Creating new log.
16:19:38.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:38.378860 5368 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVGVFlJ/build/tsan/bin/kudu
16:19:38.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskVGVFlJ/build/tsan/bin/kudu
16:19:38.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/ts-0/wal
16:19:38.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/ts-0/data
16:19:38.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:19:38.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:19:38.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:19:38.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:19:38.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:19:38.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:19:38.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/ts-0/logs
16:19:38.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/ts-0/data/info.pb
16:19:38.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:19:38.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:19:38.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:19:38.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:19:38.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:19:38.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:19:38.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:19:38.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:19:38.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
16:19:38.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:19:38.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.5.62.1:0
16:19:38.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.5.62.1
16:19:38.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.5.62.1
16:19:38.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:19:38.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.5.62.62:36133,127.5.62.61:41997,127.5.62.60:33963
16:19:38.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.5.62.20:35173
16:19:38.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:19:38.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:19:38.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
16:19:38.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:38.382913 5542 log.cc:824] T 00000000000000000000000000000000 P 82dc816b1d3142de823d77d0e689dc45: Log is configured to *not* fsync() on all Append() calls
16:19:38.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:38.472730 5542 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 82dc816b1d3142de823d77d0e689dc45: No bootstrap required, opened a new log
16:19:38.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:38.817195 5542 raft_consensus.cc:348] T 00000000000000000000000000000000 P 82dc816b1d3142de823d77d0e689dc45 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c9e99a26a2e42829ec7114980e6dc63" member_type: VOTER last_known_addr { host: "127.5.62.62" port: 36133 } } peers { permanent_uuid: "82dc816b1d3142de823d77d0e689dc45" member_type: VOTER last_known_addr { host: "127.5.62.61" port: 41997 } } peers { permanent_uuid: "65d34a1d9c7242368eae1c56211a6b90" member_type: VOTER last_known_addr { host: "127.5.62.60" port: 33963 } }
16:19:38.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:38.828533 5542 raft_consensus.cc:374] T 00000000000000000000000000000000 P 82dc816b1d3142de823d77d0e689dc45 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
16:19:38.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:38.833801 5542 raft_consensus.cc:729] T 00000000000000000000000000000000 P 82dc816b1d3142de823d77d0e689dc45 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82dc816b1d3142de823d77d0e689dc45, State: Initialized, Role: FOLLOWER
16:19:38.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:38.866639 5669 catalog_manager.cc:1259] Loaded cluster ID: c17879e963c3420d9cb1eb2bcb7eff67
16:19:38.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:38.867501 5669 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 65d34a1d9c7242368eae1c56211a6b90: loading cluster ID for follower catalog manager: success
16:19:38.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:38.867643 5542 consensus_queue.cc:260] T 00000000000000000000000000000000 P 82dc816b1d3142de823d77d0e689dc45 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c9e99a26a2e42829ec7114980e6dc63" member_type: VOTER last_known_addr { host: "127.5.62.62" port: 36133 } } peers { permanent_uuid: "82dc816b1d3142de823d77d0e689dc45" member_type: VOTER last_known_addr { host: "127.5.62.61" port: 41997 } } peers { permanent_uuid: "65d34a1d9c7242368eae1c56211a6b90" member_type: VOTER last_known_addr { host: "127.5.62.60" port: 33963 } }
16:19:38.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:38.915655 5669 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 65d34a1d9c7242368eae1c56211a6b90: acquiring CA information for follower catalog manager: success
16:19:38.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:38.954596 5683 sys_catalog.cc:455] T 00000000000000000000000000000000 P 82dc816b1d3142de823d77d0e689dc45 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c9e99a26a2e42829ec7114980e6dc63" member_type: VOTER last_known_addr { host: "127.5.62.62" port: 36133 } } peers { permanent_uuid: "82dc816b1d3142de823d77d0e689dc45" member_type: VOTER last_known_addr { host: "127.5.62.61" port: 41997 } } peers { permanent_uuid: "65d34a1d9c7242368eae1c56211a6b90" member_type: VOTER last_known_addr { host: "127.5.62.60" port: 33963 } } }
16:19:38.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:38.969298 5669 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 65d34a1d9c7242368eae1c56211a6b90: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
16:19:38.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:38.969175 5683 sys_catalog.cc:458] T 00000000000000000000000000000000 P 82dc816b1d3142de823d77d0e689dc45 [sys.catalog]: This master's current role is: FOLLOWER
16:19:38.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:38.989549 5515 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7c9e99a26a2e42829ec7114980e6dc63" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82dc816b1d3142de823d77d0e689dc45"
16:19:38.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:38.987469 5516 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7c9e99a26a2e42829ec7114980e6dc63" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "82dc816b1d3142de823d77d0e689dc45" is_pre_election: true
16:19:38.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:38.993312 5516 rpcz_store.cc:275] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:45544 (request call id 2) took 1081 ms. Trace:
16:19:38.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:38.996452 5516 rpcz_store.cc:276] 0811 16:19:37.911495 (+ 0us) service_pool.cc:170] Inserting onto call queue
16:19:38.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0811 16:19:37.911649 (+ 154us) service_pool.cc:227] Handling call
16:19:38.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0811 16:19:38.993197 (+1081548us) inbound_call.cc:173] Queueing success response
16:19:38.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
16:19:38.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:38.995826 5515 rpcz_store.cc:275] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:45544 (request call id 3) took 1058 ms. Trace:
16:19:39.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:38.999596 5515 rpcz_store.cc:276] 0811 16:19:37.937791 (+ 0us) service_pool.cc:170] Inserting onto call queue
16:19:39.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0811 16:19:37.937983 (+ 192us) service_pool.cc:227] Handling call
16:19:39.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0811 16:19:38.995738 (+1057755us) inbound_call.cc:173] Queueing success response
16:19:39.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
16:19:39.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:38.994349 5514 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 82dc816b1d3142de823d77d0e689dc45 [term 0 FOLLOWER]: Advancing to term 1
16:19:39.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:39.025038 5542 sys_catalog.cc:564] T 00000000000000000000000000000000 P 82dc816b1d3142de823d77d0e689dc45 [sys.catalog]: configured and running, proceeding with master startup.
16:19:39.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:39.039127 5492 tablet.cc:2369] T 00000000000000000000000000000000 P 82dc816b1d3142de823d77d0e689dc45: Can't schedule compaction. Clean time has not been advanced past its initial value.
16:19:39.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:39.067989 5514 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 82dc816b1d3142de823d77d0e689dc45 [term 1 FOLLOWER]: Refusing update from remote peer 7c9e99a26a2e42829ec7114980e6dc63: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
16:19:39.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:39.075114 5514 rpcz_store.cc:275] Call kudu.consensus.ConsensusService.UpdateConsensus from 127.0.0.1:45544 (request call id 4) took 1021 ms. Trace:
16:19:39.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:39.076365 5514 rpcz_store.cc:276] 0811 16:19:38.054196 (+ 0us) service_pool.cc:170] Inserting onto call queue
16:19:39.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0811 16:19:38.055192 (+ 996us) service_pool.cc:227] Handling call
16:19:39.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0811 16:19:38.986728 (+931536us) raft_consensus.cc:1457] Updating replica for 0 ops
16:19:39.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0811 16:19:39.074311 (+ 87583us) raft_consensus.cc:1683] Filling consensus response to leader.
16:19:39.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0811 16:19:39.074983 (+ 672us) inbound_call.cc:173] Queueing success response
16:19:39.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
16:19:39.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:39.079792 5682 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 7c9e99a26a2e42829ec7114980e6dc63 [LEADER]: Connected to new peer: Peer: permanent_uuid: "82dc816b1d3142de823d77d0e689dc45" member_type: VOTER last_known_addr { host: "127.5.62.61" port: 41997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
16:19:39.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:39.195782 5683 sys_catalog.cc:455] T 00000000000000000000000000000000 P 82dc816b1d3142de823d77d0e689dc45 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 7c9e99a26a2e42829ec7114980e6dc63. Latest consensus state: current_term: 1 leader_uuid: "7c9e99a26a2e42829ec7114980e6dc63" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c9e99a26a2e42829ec7114980e6dc63" member_type: VOTER last_known_addr { host: "127.5.62.62" port: 36133 } } peers { permanent_uuid: "82dc816b1d3142de823d77d0e689dc45" member_type: VOTER last_known_addr { host: "127.5.62.61" port: 41997 } } peers { permanent_uuid: "65d34a1d9c7242368eae1c56211a6b90" member_type: VOTER last_known_addr { host: "127.5.62.60" port: 33963 } } }
16:19:39.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:39.196017 5695 sys_catalog.cc:455] T 00000000000000000000000000000000 P 82dc816b1d3142de823d77d0e689dc45 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "7c9e99a26a2e42829ec7114980e6dc63" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c9e99a26a2e42829ec7114980e6dc63" member_type: VOTER last_known_addr { host: "127.5.62.62" port: 36133 } } peers { permanent_uuid: "82dc816b1d3142de823d77d0e689dc45" member_type: VOTER last_known_addr { host: "127.5.62.61" port: 41997 } } peers { permanent_uuid: "65d34a1d9c7242368eae1c56211a6b90" member_type: VOTER last_known_addr { host: "127.5.62.60" port: 33963 } } }
16:19:39.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:39.197301 5683 sys_catalog.cc:458] T 00000000000000000000000000000000 P 82dc816b1d3142de823d77d0e689dc45 [sys.catalog]: This master's current role is: FOLLOWER
16:19:39.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:39.197566 5695 sys_catalog.cc:458] T 00000000000000000000000000000000 P 82dc816b1d3142de823d77d0e689dc45 [sys.catalog]: This master's current role is: FOLLOWER
16:19:39.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:39.237246 5701 catalog_manager.cc:1259] Loaded cluster ID: c17879e963c3420d9cb1eb2bcb7eff67
16:19:39.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:39.237826 5701 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 82dc816b1d3142de823d77d0e689dc45: loading cluster ID for follower catalog manager: success
16:19:39.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:39.271473 5701 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 82dc816b1d3142de823d77d0e689dc45: acquiring CA information for follower catalog manager: success
16:19:39.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:39.298383 5701 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 82dc816b1d3142de823d77d0e689dc45: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
16:19:39.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:39.439858 5681 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:19:39.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:39.441730 5681 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:19:39.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:39.443742 5681 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:19:39.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:39.541910 5681 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:19:39.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:39.544348 5681 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.5.62.1
16:19:39.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:39.600436 5682 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7c9e99a26a2e42829ec7114980e6dc63 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "7c9e99a26a2e42829ec7114980e6dc63" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c9e99a26a2e42829ec7114980e6dc63" member_type: VOTER last_known_addr { host: "127.5.62.62" port: 36133 } } peers { permanent_uuid: "82dc816b1d3142de823d77d0e689dc45" member_type: VOTER last_known_addr { host: "127.5.62.61" port: 41997 } } peers { permanent_uuid: "65d34a1d9c7242368eae1c56211a6b90" member_type: VOTER last_known_addr { host: "127.5.62.60" port: 33963 } } }
16:19:39.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:39.601861 5682 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7c9e99a26a2e42829ec7114980e6dc63 [sys.catalog]: This master's current role is: LEADER
16:19:39.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:39.688117 5681 tablet_server_runner.cc:78] Tablet server non-default flags:
16:19:39.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:19:39.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.5.62.20:35173
16:19:39.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:19:39.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:19:39.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/ts-0/data
16:19:39.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/ts-0/wal
16:19:39.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:19:39.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:19:39.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.5.62.1:0
16:19:39.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:19:39.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:19:39.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:19:39.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/ts-0/data/info.pb
16:19:39.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.5.62.1
16:19:39.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:19:39.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.5.62.62:36133,127.5.62.61:41997,127.5.62.60:33963
16:19:39.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:19:39.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:19:39.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:19:39.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:19:39.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:19:39.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.5.62.1
16:19:39.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/ts-0/logs
16:19:39.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:19:39.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:19:39.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:19:39.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
16:19:39.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
16:19:39.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
16:19:39.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:19:39.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 15:12:52 UTC on ccycloud.quasar-tzrfpd.root.comops.site
16:19:39.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4580143
16:19:39.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:19:39.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:39.705780 5681 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:19:39.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:39.710747 5681 file_cache.cc:493] Constructed file cache file cache with capacity 419430
16:19:39.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:39.762732 5715 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:41.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:39.773015 5681 server_base.cc:1034] running on GCE node
16:19:41.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:39.778535 5714 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:41.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:39.778905 5719 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:41.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:41.305962 5681 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:19:41.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:41.321313 5681 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:19:41.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:41.324990 5681 hybrid_clock.cc:648] HybridClock initialized: now 1786465181324942 us; error 55 us; skew 500 ppm
16:19:41.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:41.326399 5681 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:19:41.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:41.365062 5681 webserver.cc:460] Webserver started at http://127.5.62.1:33313/ using document root <none> and password file <none>
16:19:41.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:41.366870 5681 fs_manager.cc:359] Metadata directory not provided
16:19:41.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:41.367396 5681 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:19:41.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:41.368217 5681 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
16:19:41.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:41.375782 5681 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/ts-0/data/instance:
16:19:41.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f1af3376e1f64a83998baf05269d0bb1"
16:19:41.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 16:19:41 on dist-test-slave-cbbz"
16:19:41.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:41.377940 5681 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/ts-0/wal/instance:
16:19:41.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f1af3376e1f64a83998baf05269d0bb1"
16:19:41.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 16:19:41 on dist-test-slave-cbbz"
16:19:41.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:41.394019 5681 fs_manager.cc:691] Time spent creating directory manager: real 0.015s user 0.009s sys 0.004s
16:19:41.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:41.407842 5729 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:19:41.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:41.409502 5681 fs_manager.cc:725] Time spent opening block manager: real 0.011s user 0.010s sys 0.000s
16:19:41.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:41.410142 5681 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/ts-0/data,/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/ts-0/wal
16:19:41.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f1af3376e1f64a83998baf05269d0bb1"
16:19:41.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 16:19:41 on dist-test-slave-cbbz"
16:19:41.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:41.410921 5681 fs_report.cc:383] FS layout report
16:19:41.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:19:41.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/ts-0/wal
16:19:41.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/ts-0/wal
16:19:41.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/ts-0/data/data
16:19:41.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:19:41.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:19:41.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:19:41.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:19:41.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:19:41.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:19:41.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:19:41.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:19:41.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:19:41.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:19:41.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:19:41.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:41.492646 5681 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
16:19:41.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:41.495082 5681 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:19:41.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:41.495936 5681 kserver.cc:163] Server-wide thread pool size limit: 3276
16:19:41.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:41.505432 5681 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
16:19:41.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:41.521759 5681 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
16:19:41.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:41.522362 5681 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.001s user 0.001s sys 0.000s
16:19:41.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:41.523181 5681 ts_tablet_manager.cc:610] Registered 0 tablets
16:19:41.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:41.523713 5681 ts_tablet_manager.cc:589] Time spent register tablets: real 0.001s user 0.000s sys 0.000s
16:19:41.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:41.813688 5681 rpc_server.cc:307] RPC server started. Bound to: 127.5.62.1:38841
16:19:41.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:41.813805 5852 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.5.62.1:38841 every 8 connection(s)
16:19:41.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:41.817024 5681 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/ts-0/data/info.pb
16:19:41.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:41.826953 5368 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVGVFlJ/build/tsan/bin/kudu as pid 5681
16:19:41.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:41.827461 5368 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/ts-0/wal/instance
16:19:41.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:41.855701 5368 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVGVFlJ/build/tsan/bin/kudu
16:19:41.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskVGVFlJ/build/tsan/bin/kudu
16:19:41.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/ts-1/wal
16:19:41.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/ts-1/data
16:19:41.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:19:41.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:19:41.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:19:41.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:19:41.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:19:41.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:19:41.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/ts-1/logs
16:19:41.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/ts-1/data/info.pb
16:19:41.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:19:41.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:19:41.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:19:41.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:19:41.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:19:41.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:19:41.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:19:41.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:19:41.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
16:19:41.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:19:41.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.5.62.2:0
16:19:41.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.5.62.2
16:19:41.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.5.62.2
16:19:41.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:19:41.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.5.62.62:36133,127.5.62.61:41997,127.5.62.60:33963
16:19:41.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.5.62.20:35173
16:19:41.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:19:41.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:19:41.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
16:19:41.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:41.911811 5854 heartbeater.cc:347] Connected to a master server at 127.5.62.61:41997
16:19:41.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:41.912346 5854 heartbeater.cc:464] Registering TS with master...
16:19:41.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:41.913552 5854 heartbeater.cc:511] Master 127.5.62.61:41997 requested a full tablet report, sending...
16:19:41.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:41.915014 5853 heartbeater.cc:347] Connected to a master server at 127.5.62.60:33963
16:19:41.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:41.915347 5853 heartbeater.cc:464] Registering TS with master...
16:19:41.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:41.915459 5860 heartbeater.cc:347] Connected to a master server at 127.5.62.62:36133
16:19:41.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:41.915764 5860 heartbeater.cc:464] Registering TS with master...
16:19:41.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:41.916180 5853 heartbeater.cc:511] Master 127.5.62.60:33963 requested a full tablet report, sending...
16:19:41.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:41.916601 5860 heartbeater.cc:511] Master 127.5.62.62:36133 requested a full tablet report, sending...
16:19:41.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:41.917104 5504 ts_manager.cc:194] Registered new tserver with Master: f1af3376e1f64a83998baf05269d0bb1 (127.5.62.1:38841)
16:19:41.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:41.919095 5419 ts_manager.cc:194] Registered new tserver with Master: f1af3376e1f64a83998baf05269d0bb1 (127.5.62.1:38841)
16:19:41.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:41.919869 5590 ts_manager.cc:194] Registered new tserver with Master: f1af3376e1f64a83998baf05269d0bb1 (127.5.62.1:38841)
16:19:41.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:41.922081 5419 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.5.62.1:60399
16:19:42.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:42.331619 5865 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:19:42.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:42.332746 5865 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:19:42.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:42.333871 5865 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:19:42.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:42.374266 5865 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:19:42.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:42.375118 5865 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.5.62.2
16:19:42.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:42.410959 5865 tablet_server_runner.cc:78] Tablet server non-default flags:
16:19:42.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:19:42.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.5.62.20:35173
16:19:42.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:19:42.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:19:42.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/ts-1/data
16:19:42.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/ts-1/wal
16:19:42.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:19:42.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:19:42.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.5.62.2:0
16:19:42.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:19:42.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:19:42.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:19:42.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/ts-1/data/info.pb
16:19:42.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.5.62.2
16:19:42.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:19:42.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.5.62.62:36133,127.5.62.61:41997,127.5.62.60:33963
16:19:42.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:19:42.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:19:42.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:19:42.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:19:42.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:19:42.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.5.62.2
16:19:42.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/ts-1/logs
16:19:42.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:19:42.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:19:42.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:19:42.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
16:19:42.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
16:19:42.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
16:19:42.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:19:42.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 15:12:52 UTC on ccycloud.quasar-tzrfpd.root.comops.site
16:19:42.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4580143
16:19:42.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:19:42.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:42.412370 5865 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:19:42.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:42.414222 5865 file_cache.cc:493] Constructed file cache file cache with capacity 419430
16:19:42.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:42.437019 5875 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:42.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:42.926957 5860 heartbeater.cc:503] Master 127.5.62.62:36133 was elected leader, sending a full tablet report...
16:19:43.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:43.341243 5847 debug-util.cc:398] Leaking SignalData structure 0x7b08000af060 after lost signal to thread 5703
16:19:43.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:43.342111 5847 debug-util.cc:398] Leaking SignalData structure 0x7b08000aff40 after lost signal to thread 5852
16:19:43.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:42.438544 5877 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:43.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:42.440012 5865 server_base.cc:1034] running on GCE node
16:19:43.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:42.437009 5874 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:43.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:43.666339 5865 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:19:43.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:43.669209 5865 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:19:43.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:43.670686 5865 hybrid_clock.cc:648] HybridClock initialized: now 1786465183670656 us; error 61 us; skew 500 ppm
16:19:43.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:43.671749 5865 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:19:43.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:43.685985 5865 webserver.cc:460] Webserver started at http://127.5.62.2:33085/ using document root <none> and password file <none>
16:19:43.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:43.686978 5865 fs_manager.cc:359] Metadata directory not provided
16:19:43.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:43.687176 5865 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:19:43.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:43.687579 5865 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
16:19:43.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:43.692783 5865 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/ts-1/data/instance:
16:19:43.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6a8a46edc0484842b405183c1ff53dba"
16:19:43.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 16:19:43 on dist-test-slave-cbbz"
16:19:43.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:43.693953 5865 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/ts-1/wal/instance:
16:19:43.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6a8a46edc0484842b405183c1ff53dba"
16:19:43.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 16:19:43 on dist-test-slave-cbbz"
16:19:43.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:43.701930 5865 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.000s sys 0.008s
16:19:43.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:43.708702 5886 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:19:43.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:43.709945 5865 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.004s sys 0.000s
16:19:43.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:43.710244 5865 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/ts-1/data,/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/ts-1/wal
16:19:43.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6a8a46edc0484842b405183c1ff53dba"
16:19:43.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 16:19:43 on dist-test-slave-cbbz"
16:19:43.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:43.710574 5865 fs_report.cc:383] FS layout report
16:19:43.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:19:43.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/ts-1/wal
16:19:43.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/ts-1/wal
16:19:43.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/ts-1/data/data
16:19:43.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:19:43.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:19:43.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:19:43.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:19:43.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:19:43.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:19:43.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:19:43.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:19:43.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:19:43.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:19:43.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:19:43.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:43.758586 5865 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
16:19:43.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:43.760020 5865 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:19:43.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:43.760524 5865 kserver.cc:163] Server-wide thread pool size limit: 3276
16:19:43.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:43.763541 5865 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
16:19:43.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:43.768640 5865 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
16:19:43.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:43.768847 5865 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
16:19:43.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:43.769048 5865 ts_tablet_manager.cc:610] Registered 0 tablets
16:19:43.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:43.769181 5865 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
16:19:44.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:44.065971 5865 rpc_server.cc:307] RPC server started. Bound to: 127.5.62.2:36661
16:19:44.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:44.066041 6008 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.5.62.2:36661 every 8 connection(s)
16:19:44.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:44.069979 5865 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/ts-1/data/info.pb
16:19:44.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:44.073643 5368 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVGVFlJ/build/tsan/bin/kudu as pid 5865
16:19:44.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:44.074340 5368 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/ts-1/wal/instance
16:19:44.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:44.086704 5368 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVGVFlJ/build/tsan/bin/kudu
16:19:44.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskVGVFlJ/build/tsan/bin/kudu
16:19:44.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/ts-2/wal
16:19:44.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/ts-2/data
16:19:44.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:19:44.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:19:44.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:19:44.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:19:44.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:19:44.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:19:44.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/ts-2/logs
16:19:44.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/ts-2/data/info.pb
16:19:44.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:19:44.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:19:44.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:19:44.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:19:44.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:19:44.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:19:44.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:19:44.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:19:44.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
16:19:44.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:19:44.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.5.62.3:0
16:19:44.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.5.62.3
16:19:44.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.5.62.3
16:19:44.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:19:44.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.5.62.62:36133,127.5.62.61:41997,127.5.62.60:33963
16:19:44.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.5.62.20:35173
16:19:44.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:19:44.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:19:44.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
16:19:44.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:44.144908 6010 heartbeater.cc:347] Connected to a master server at 127.5.62.61:41997
16:19:44.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:44.146112 6010 heartbeater.cc:464] Registering TS with master...
16:19:44.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:44.147491 6010 heartbeater.cc:511] Master 127.5.62.61:41997 requested a full tablet report, sending...
16:19:44.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:44.151921 5504 ts_manager.cc:194] Registered new tserver with Master: 6a8a46edc0484842b405183c1ff53dba (127.5.62.2:36661)
16:19:44.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:44.154452 6009 heartbeater.cc:347] Connected to a master server at 127.5.62.60:33963
16:19:44.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:44.154815 6009 heartbeater.cc:464] Registering TS with master...
16:19:44.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:44.155761 6009 heartbeater.cc:511] Master 127.5.62.60:33963 requested a full tablet report, sending...
16:19:44.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:44.158586 5590 ts_manager.cc:194] Registered new tserver with Master: 6a8a46edc0484842b405183c1ff53dba (127.5.62.2:36661)
16:19:44.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:44.161938 6015 heartbeater.cc:347] Connected to a master server at 127.5.62.62:36133
16:19:44.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:44.162254 6015 heartbeater.cc:464] Registering TS with master...
16:19:44.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:44.163169 6015 heartbeater.cc:511] Master 127.5.62.62:36133 requested a full tablet report, sending...
16:19:44.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:44.165463 5419 ts_manager.cc:194] Registered new tserver with Master: 6a8a46edc0484842b405183c1ff53dba (127.5.62.2:36661)
16:19:44.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:44.167248 5419 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.5.62.2:36751
16:19:44.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:44.476944 6014 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:19:44.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:44.477561 6014 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:19:44.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:44.478276 6014 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:19:44.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:44.511421 6014 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:19:44.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:44.512355 6014 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.5.62.3
16:19:44.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:44.549865 6014 tablet_server_runner.cc:78] Tablet server non-default flags:
16:19:44.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:19:44.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.5.62.20:35173
16:19:44.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:19:44.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:19:44.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/ts-2/data
16:19:44.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/ts-2/wal
16:19:44.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:19:44.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:19:44.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.5.62.3:0
16:19:44.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:19:44.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:19:44.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:19:44.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/ts-2/data/info.pb
16:19:44.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.5.62.3
16:19:44.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:19:44.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.5.62.62:36133,127.5.62.61:41997,127.5.62.60:33963
16:19:44.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:19:44.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:19:44.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:19:44.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:19:44.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:19:44.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.5.62.3
16:19:44.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/ts-2/logs
16:19:44.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:19:44.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:19:44.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:19:44.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
16:19:44.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
16:19:44.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
16:19:44.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:19:44.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 15:12:52 UTC on ccycloud.quasar-tzrfpd.root.comops.site
16:19:44.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4580143
16:19:44.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:19:44.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:44.551105 6014 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:19:44.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:44.552968 6014 file_cache.cc:493] Constructed file cache file cache with capacity 419430
16:19:44.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:44.576773 6027 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:44.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:44.979938 5454 debug-util.cc:398] Leaking SignalData structure 0x7b08000675c0 after lost signal to thread 5383
16:19:45.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:45.171130 6015 heartbeater.cc:503] Master 127.5.62.62:36133 was elected leader, sending a full tablet report...
16:19:46.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:44.577970 6030 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:46.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:44.576845 6028 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:46.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:46.134243 6029 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1554 milliseconds
16:19:46.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:46.134352 6014 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:19:46.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:46.135668 6014 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:19:46.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:46.138965 6014 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:19:46.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:46.140513 6014 hybrid_clock.cc:648] HybridClock initialized: now 1786465186140478 us; error 55 us; skew 500 ppm
16:19:46.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:46.141360 6014 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:19:46.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:46.149314 6014 webserver.cc:460] Webserver started at http://127.5.62.3:38135/ using document root <none> and password file <none>
16:19:46.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:46.150404 6014 fs_manager.cc:359] Metadata directory not provided
16:19:46.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:46.150667 6014 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:19:46.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:46.151234 6014 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
16:19:46.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:46.156368 6014 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/ts-2/data/instance:
16:19:46.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "141587dbcb5541e9b0038e2f74a5a045"
16:19:46.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 16:19:46 on dist-test-slave-cbbz"
16:19:46.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:46.157516 6014 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/ts-2/wal/instance:
16:19:46.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "141587dbcb5541e9b0038e2f74a5a045"
16:19:46.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 16:19:46 on dist-test-slave-cbbz"
16:19:46.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:46.166335 6014 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.005s sys 0.004s
16:19:46.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:46.173177 6038 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:19:46.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:46.174398 6014 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.003s
16:19:46.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:46.174757 6014 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/ts-2/data,/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/ts-2/wal
16:19:46.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "141587dbcb5541e9b0038e2f74a5a045"
16:19:46.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 16:19:46 on dist-test-slave-cbbz"
16:19:46.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:46.175100 6014 fs_report.cc:383] FS layout report
16:19:46.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:19:46.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/ts-2/wal
16:19:46.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/ts-2/wal
16:19:46.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/ts-2/data/data
16:19:46.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:19:46.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:19:46.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:19:46.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:19:46.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:19:46.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:19:46.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:19:46.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:19:46.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:19:46.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:19:46.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:19:46.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:46.217474 6014 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
16:19:46.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:46.219033 6014 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:19:46.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:46.219570 6014 kserver.cc:163] Server-wide thread pool size limit: 3276
16:19:46.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:46.222955 6014 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
16:19:46.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:46.227998 6014 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
16:19:46.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:46.228255 6014 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.001s sys 0.000s
16:19:46.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:46.228631 6014 ts_tablet_manager.cc:610] Registered 0 tablets
16:19:46.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:46.228842 6014 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
16:19:46.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:46.414461 6014 rpc_server.cc:307] RPC server started. Bound to: 127.5.62.3:43633
16:19:46.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:46.414574 6163 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.5.62.3:43633 every 8 connection(s)
16:19:46.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:46.417320 6014 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/ts-2/data/info.pb
16:19:46.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:46.421339 5368 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVGVFlJ/build/tsan/bin/kudu as pid 6014
16:19:46.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:46.421911 5368 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster9038909321259978813/ts-2/wal/instance
16:19:46.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:46.484371 6165 heartbeater.cc:347] Connected to a master server at 127.5.62.60:33963
16:19:46.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:46.484872 6165 heartbeater.cc:464] Registering TS with master...
16:19:46.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:46.486112 6165 heartbeater.cc:511] Master 127.5.62.60:33963 requested a full tablet report, sending...
16:19:46.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:46.487607 6166 heartbeater.cc:347] Connected to a master server at 127.5.62.61:41997
16:19:46.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:46.487986 6166 heartbeater.cc:464] Registering TS with master...
16:19:46.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:46.488897 6166 heartbeater.cc:511] Master 127.5.62.61:41997 requested a full tablet report, sending...
16:19:46.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:46.490883 5590 ts_manager.cc:194] Registered new tserver with Master: 141587dbcb5541e9b0038e2f74a5a045 (127.5.62.3:43633)
16:19:46.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:46.491715 5504 ts_manager.cc:194] Registered new tserver with Master: 141587dbcb5541e9b0038e2f74a5a045 (127.5.62.3:43633)
16:19:46.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:46.497361 6167 heartbeater.cc:347] Connected to a master server at 127.5.62.62:36133
16:19:46.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:46.497682 6167 heartbeater.cc:464] Registering TS with master...
16:19:46.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:46.498543 6167 heartbeater.cc:511] Master 127.5.62.62:36133 requested a full tablet report, sending...
16:19:46.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:46.500703 5419 ts_manager.cc:194] Registered new tserver with Master: 141587dbcb5541e9b0038e2f74a5a045 (127.5.62.3:43633)
16:19:46.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:46.502986 5419 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.5.62.3:36965
16:19:46.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:46.513588 5368 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
16:19:46.526 [DEBUG - main] (MiniKuduCluster.java:195) Response:
16:19:46.527 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
16:19:46.538 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.5.62.62"
port: 36133
}
bound_http_address {
host: "127.5.62.62"
port: 44321
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.5.62.61"
port: 41997
}
bound_http_address {
host: "127.5.62.61"
port: 46613
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.5.62.60"
port: 33963
}
bound_http_address {
host: "127.5.62.60"
port: 37159
}
}
}
16:19:46.544 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
16:19:46.548 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.5.62.1"
port: 38841
}
bound_http_address {
host: "127.5.62.1"
port: 33313
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.5.62.2"
port: 36661
}
bound_http_address {
host: "127.5.62.2"
port: 33085
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.5.62.3"
port: 43633
}
bound_http_address {
host: "127.5.62.3"
port: 38135
}
}
}
16:19:46.548 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
16:19:46.781 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
16:19:46.925 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskVGVFlJ/test-tmp
16:19:46.926 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
16:19:46.926 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskVGVFlJ/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
16:19:46.930 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
16:19:47.310 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:19:47.311 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
16:19:47.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:47.316363 6179 mini_oidc.cc:152] Starting JWKS server
16:19:47.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:47.320627 6179 webserver.cc:460] Webserver started at http://127.0.0.1:44069/ using document root <none> and password file <none>
16:19:47.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:47.321173 6179 mini_oidc.cc:189] Starting OIDC Discovery server
16:19:47.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:47.323374 6179 webserver.cc:460] Webserver started at http://127.0.0.1:41143/ using document root <none> and password file <none>
16:19:47.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
16:19:47.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/krb5kdc/principal' for realm 'KRBTEST.COM',
16:19:47.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
16:19:47.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 16:19:47 dist-test-slave-cbbz krb5kdc[6192](info): setting up network...
16:19:47.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
16:19:47.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 16:19:47 dist-test-slave-cbbz krb5kdc[6192](info): set up 2 sockets
16:19:47.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 16:19:47 dist-test-slave-cbbz krb5kdc[6192](info): commencing operation
16:19:47.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
16:19:47.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:47.506935 6167 heartbeater.cc:503] Master 127.5.62.62:36133 was elected leader, sending a full tablet report...
16:19:48.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:48.637132 6179 mini_kdc.cc:121] Time spent starting KDC: real 1.286s user 0.010s sys 0.025s
16:19:48.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
16:19:48.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
16:19:48.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
16:19:48.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
16:19:48.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
16:19:48.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
16:19:48.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
16:19:48.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
16:19:48.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
16:19:48.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
16:19:48.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/krb5kdc/test-user.keytab.
16:19:48.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/krb5kdc/test-user.keytab.
16:19:48.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 16:19:48 dist-test-slave-cbbz krb5kdc[6192](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786465188, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:19:48.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
16:19:48.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-11T16:19:48Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
16:19:48.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-11T16:19:48Z Disabled control of system clock
16:19:48.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.6.8.254@KRBTEST.COM; defaulting to no policy
16:19:48.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:19:48.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.6.8.254@KRBTEST.COM" created.
16:19:48.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:19:48.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.6.8.254 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/krb5kdc/kudu_127.6.8.254.keytab.
16:19:48.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.6.8.254 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/krb5kdc/kudu_127.6.8.254.keytab.
16:19:48.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:48.849740 6179 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVGVFlJ/build/tsan/bin/kudu
16:19:48.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskVGVFlJ/build/tsan/bin/kudu
16:19:48.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-0/wal
16:19:48.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-0/data
16:19:48.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:19:48.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:19:48.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:19:48.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:19:48.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:19:48.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:19:48.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-0/logs
16:19:48.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-0/data/info.pb
16:19:48.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:19:48.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:19:48.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:19:48.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:19:48.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:19:48.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:19:48.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:19:48.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:19:48.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:19:48.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:19:48.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:19:48.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:19:48.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.8.254:34321
16:19:48.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.8.254
16:19:48.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:19:48.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/krb5kdc/kudu_127.6.8.254.keytab
16:19:48.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.6.8.254
16:19:48.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:19:48.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:19:48.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:19:48.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.8.212:33651
16:19:48.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:19:48.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:19:48.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:19:48.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:19:48.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.8.254:34321,127.6.8.253:41923,127.6.8.252:39609
16:19:48.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:19:48.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44069/jwks.json
16:19:48.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:19:49.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:49.163654 6205 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:19:49.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:49.164258 6205 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:19:49.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:49.164851 6205 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:19:49.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:49.198159 6205 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
16:19:49.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:49.198406 6205 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:19:49.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:49.198678 6205 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:19:49.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:49.198912 6205 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:19:49.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:49.199134 6205 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:19:49.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:49.199402 6205 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:19:49.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:49.199558 6205 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44069/jwks.json
16:19:49.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:49.236137 6205 master_runner.cc:386] Master server non-default flags:
16:19:49.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:19:49.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.8.212:33651
16:19:49.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:19:49.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:19:49.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-0/data
16:19:49.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-0/wal
16:19:49.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
16:19:49.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:19:49.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.8.254:34321,127.6.8.253:41923,127.6.8.252:39609
16:19:49.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/krb5kdc/kudu_127.6.8.254.keytab
16:19:49.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.6.8.254
16:19:49.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:19:49.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:19:49.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:19:49.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.8.254:34321
16:19:49.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:19:49.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:19:49.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:19:49.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44069/jwks.json
16:19:49.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:19:49.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:19:49.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:19:49.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:19:49.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:19:49.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-0/data/info.pb
16:19:49.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.8.254
16:19:49.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:19:49.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:19:49.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:19:49.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:19:49.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:19:49.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:19:49.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-0/logs
16:19:49.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:19:49.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:19:49.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:19:49.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:19:49.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
16:19:49.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
16:19:49.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:19:49.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 15:12:52 UTC on ccycloud.quasar-tzrfpd.root.comops.site
16:19:49.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4580143
16:19:49.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:19:49.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:49.237490 6205 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:19:49.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:49.238876 6205 file_cache.cc:493] Constructed file cache file cache with capacity 419430
16:19:49.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:49.260520 6213 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:50.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:49.260541 6216 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:50.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:49.260928 6212 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:50.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:49.263025 6205 server_base.cc:1034] running on GCE node
16:19:50.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:50.495929 6205 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:19:50.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:50.498694 6205 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:19:50.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:50.500057 6205 hybrid_clock.cc:648] HybridClock initialized: now 1786465190500052 us; error 105 us; skew 500 ppm
16:19:50.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:50.500929 6205 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:19:50.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 16:19:50 dist-test-slave-cbbz krb5kdc[6192](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786465190, etypes {rep=17 tkt=17 ses=17}, kudu/127.6.8.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:19:50.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:50.511751 6205 init.cc:377] Logged in from keytab as kudu/127.6.8.254@KRBTEST.COM (short username kudu)
16:19:50.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:50.517261 6205 webserver.cc:460] Webserver started at http://127.6.8.254:42747/ using document root <none> and password file <none>
16:19:50.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:50.518275 6205 fs_manager.cc:359] Metadata directory not provided
16:19:50.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:50.518560 6205 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:19:50.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:50.519070 6205 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
16:19:50.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:50.524894 6205 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-0/data/instance:
16:19:50.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9636eb75872e49cb90a026bad879c251"
16:19:50.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 16:19:50 on dist-test-slave-cbbz"
16:19:50.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:50.526172 6205 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-0/wal/instance:
16:19:50.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9636eb75872e49cb90a026bad879c251"
16:19:50.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 16:19:50 on dist-test-slave-cbbz"
16:19:50.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:50.534024 6205 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.004s
16:19:50.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:50.539938 6226 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:19:50.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:50.541020 6205 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.001s sys 0.002s
16:19:50.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:50.541333 6205 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-0/data,/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-0/wal
16:19:50.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9636eb75872e49cb90a026bad879c251"
16:19:50.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 16:19:50 on dist-test-slave-cbbz"
16:19:50.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:50.541618 6205 fs_report.cc:383] FS layout report
16:19:50.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:19:50.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-0/wal
16:19:50.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-0/wal
16:19:50.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-0/data/data
16:19:50.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:19:50.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:19:50.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:19:50.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:19:50.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:19:50.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:19:50.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:19:50.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:19:50.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:19:50.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:19:50.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:19:50.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:50.581337 6205 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
16:19:50.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:50.587031 6205 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:19:50.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:50.587453 6205 kserver.cc:163] Server-wide thread pool size limit: 3276
16:19:50.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:50.670444 6205 rpc_server.cc:307] RPC server started. Bound to: 127.6.8.254:34321
16:19:50.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:50.670493 6282 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.8.254:34321 every 8 connection(s)
16:19:50.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:50.675328 6205 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-0/data/info.pb
16:19:50.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:50.681931 6179 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVGVFlJ/build/tsan/bin/kudu as pid 6205
16:19:50.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:50.682569 6179 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-0/wal/instance
16:19:50.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:50.682560 6283 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
16:19:50.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:50.703037 6283 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.8.254" port: 34321 } has no permanent_uuid. Determining permanent_uuid...
16:19:50.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.6.8.253@KRBTEST.COM; defaulting to no policy
16:19:50.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:19:50.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.6.8.253@KRBTEST.COM" created.
16:19:50.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 16:19:50 dist-test-slave-cbbz krb5kdc[6192](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786465190, etypes {rep=17 tkt=17 ses=17}, kudu/127.6.8.254@KRBTEST.COM for kudu/127.6.8.254@KRBTEST.COM
16:19:50.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:19:50.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.6.8.253 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/krb5kdc/kudu_127.6.8.253.keytab.
16:19:50.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.6.8.253 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/krb5kdc/kudu_127.6.8.253.keytab.
16:19:50.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:50.744576 6179 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVGVFlJ/build/tsan/bin/kudu
16:19:50.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskVGVFlJ/build/tsan/bin/kudu
16:19:50.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-1/wal
16:19:50.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-1/data
16:19:50.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:19:50.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:19:50.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:19:50.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:19:50.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:19:50.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:19:50.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-1/logs
16:19:50.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-1/data/info.pb
16:19:50.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:19:50.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:19:50.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:19:50.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:19:50.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:19:50.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:19:50.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:19:50.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:19:50.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:19:50.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:19:50.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:19:50.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:19:50.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.8.253:41923
16:19:50.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.8.253
16:19:50.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:19:50.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/krb5kdc/kudu_127.6.8.253.keytab
16:19:50.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.6.8.253
16:19:50.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:19:50.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:19:50.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:19:50.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.8.212:33651
16:19:50.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:19:50.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:19:50.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:19:50.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:19:50.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.8.254:34321,127.6.8.253:41923,127.6.8.252:39609
16:19:50.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:19:50.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44069/jwks.json
16:19:50.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:19:50.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:50.751768 6283 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.8.253" port: 41923 } has no permanent_uuid. Determining permanent_uuid...
16:19:50.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:50.754211 6228 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.8.253:41923: connect: Connection refused (error 111)
16:19:50.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:50.758381 6283 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.8.253:41923: Network error: Client connection negotiation failed: client connection to 127.6.8.253:41923: connect: Connection refused (error 111)
16:19:50.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:50.807832 6283 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.8.253" port: 41923 } attempt: 1
16:19:50.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:50.811492 6283 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.8.253:41923: Network error: Client connection negotiation failed: client connection to 127.6.8.253:41923: connect: Connection refused (error 111)
16:19:50.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:50.879942 6283 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.8.253" port: 41923 } attempt: 2
16:19:50.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:50.883752 6283 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.8.253:41923: Network error: Client connection negotiation failed: client connection to 127.6.8.253:41923: connect: Connection refused (error 111)
16:19:50.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:50.975339 6283 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.8.253" port: 41923 } attempt: 3
16:19:50.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:50.979945 6283 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.8.253:41923: Network error: Client connection negotiation failed: client connection to 127.6.8.253:41923: connect: Connection refused (error 111)
16:19:51.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:51.067600 6289 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:19:51.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:51.068183 6289 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:19:51.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:51.068787 6289 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:19:51.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:51.100539 6289 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
16:19:51.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:51.100829 6289 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:19:51.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:51.101100 6289 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:19:51.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:51.101300 6289 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:19:51.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:51.101488 6289 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:19:51.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:51.101673 6289 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:19:51.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:51.101818 6289 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44069/jwks.json
16:19:51.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:51.123399 6283 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.8.253" port: 41923 } attempt: 4
16:19:51.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:51.127308 6283 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.8.253:41923: Network error: Client connection negotiation failed: client connection to 127.6.8.253:41923: connect: Connection refused (error 111)
16:19:51.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:51.139853 6289 master_runner.cc:386] Master server non-default flags:
16:19:51.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:19:51.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.8.212:33651
16:19:51.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:19:51.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:19:51.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-1/data
16:19:51.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-1/wal
16:19:51.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
16:19:51.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:19:51.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.8.254:34321,127.6.8.253:41923,127.6.8.252:39609
16:19:51.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/krb5kdc/kudu_127.6.8.253.keytab
16:19:51.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.6.8.253
16:19:51.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:19:51.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:19:51.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:19:51.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.8.253:41923
16:19:51.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:19:51.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:19:51.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:19:51.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44069/jwks.json
16:19:51.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:19:51.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:19:51.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:19:51.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:19:51.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:19:51.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-1/data/info.pb
16:19:51.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.8.253
16:19:51.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:19:51.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:19:51.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:19:51.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:19:51.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:19:51.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:19:51.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-1/logs
16:19:51.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:19:51.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:19:51.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:19:51.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:19:51.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
16:19:51.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
16:19:51.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:19:51.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 15:12:52 UTC on ccycloud.quasar-tzrfpd.root.comops.site
16:19:51.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4580143
16:19:51.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:19:51.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:51.141168 6289 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:19:51.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:51.142354 6289 file_cache.cc:493] Constructed file cache file cache with capacity 419430
16:19:51.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:51.160284 6301 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:51.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:51.426762 6283 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.8.253" port: 41923 } attempt: 5
16:19:51.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:51.432214 6283 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.8.253:41923: Network error: Client connection negotiation failed: client connection to 127.6.8.253:41923: connect: Connection refused (error 111)
16:19:51.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:51.996311 6283 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.8.253" port: 41923 } attempt: 6
16:19:52.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:52.003931 6283 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.8.253:41923: Network error: Client connection negotiation failed: client connection to 127.6.8.253:41923: connect: Connection refused (error 111)
16:19:52.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:51.160292 6298 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:52.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:51.160909 6297 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:52.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:51.162788 6289 server_base.cc:1034] running on GCE node
16:19:52.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:52.433333 6289 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:19:52.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:52.436518 6289 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:19:52.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:52.437932 6289 hybrid_clock.cc:648] HybridClock initialized: now 1786465192437885 us; error 43 us; skew 500 ppm
16:19:52.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:52.438866 6289 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:19:52.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 16:19:52 dist-test-slave-cbbz krb5kdc[6192](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786465192, etypes {rep=17 tkt=17 ses=17}, kudu/127.6.8.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:19:52.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:52.445024 6289 init.cc:377] Logged in from keytab as kudu/127.6.8.253@KRBTEST.COM (short username kudu)
16:19:52.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:52.450611 6289 webserver.cc:460] Webserver started at http://127.6.8.253:41921/ using document root <none> and password file <none>
16:19:52.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:52.451534 6289 fs_manager.cc:359] Metadata directory not provided
16:19:52.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:52.451897 6289 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:19:52.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:52.452447 6289 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
16:19:52.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:52.457347 6289 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-1/data/instance:
16:19:52.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a297c0bf389d4a5ca444bd373be38e48"
16:19:52.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 16:19:52 on dist-test-slave-cbbz"
16:19:52.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:52.458458 6289 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-1/wal/instance:
16:19:52.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a297c0bf389d4a5ca444bd373be38e48"
16:19:52.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 16:19:52 on dist-test-slave-cbbz"
16:19:52.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:52.465919 6289 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.001s sys 0.007s
16:19:52.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:52.471235 6314 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:19:52.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:52.472277 6289 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
16:19:52.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:52.472625 6289 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-1/data,/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-1/wal
16:19:52.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a297c0bf389d4a5ca444bd373be38e48"
16:19:52.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 16:19:52 on dist-test-slave-cbbz"
16:19:52.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:52.472954 6289 fs_report.cc:383] FS layout report
16:19:52.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:19:52.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-1/wal
16:19:52.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-1/wal
16:19:52.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-1/data/data
16:19:52.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:19:52.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:19:52.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:19:52.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:19:52.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:19:52.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:19:52.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:19:52.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:19:52.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:19:52.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:19:52.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:19:52.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:52.515820 6289 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
16:19:52.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:52.521960 6289 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:19:52.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:52.522513 6289 kserver.cc:163] Server-wide thread pool size limit: 3276
16:19:52.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:52.604277 6289 rpc_server.cc:307] RPC server started. Bound to: 127.6.8.253:41923
16:19:52.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:52.604421 6372 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.8.253:41923 every 8 connection(s)
16:19:52.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:52.607263 6289 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-1/data/info.pb
16:19:52.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:52.611949 6179 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVGVFlJ/build/tsan/bin/kudu as pid 6289
16:19:52.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:52.612474 6179 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-1/wal/instance
16:19:52.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:52.613634 6374 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
16:19:52.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:52.616626 6279 debug-util.cc:398] Leaking SignalData structure 0x7b0800073060 after lost signal to thread 6206
16:19:52.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:52.617282 6279 debug-util.cc:398] Leaking SignalData structure 0x7b08000737c0 after lost signal to thread 6282
16:19:52.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:52.631754 6374 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.8.254" port: 34321 } has no permanent_uuid. Determining permanent_uuid...
16:19:52.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.6.8.252@KRBTEST.COM; defaulting to no policy
16:19:52.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:19:52.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.6.8.252@KRBTEST.COM" created.
16:19:52.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 16:19:52 dist-test-slave-cbbz krb5kdc[6192](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786465192, etypes {rep=17 tkt=17 ses=17}, kudu/127.6.8.253@KRBTEST.COM for kudu/127.6.8.254@KRBTEST.COM
16:19:52.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:19:52.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.6.8.252 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/krb5kdc/kudu_127.6.8.252.keytab.
16:19:52.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.6.8.252 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/krb5kdc/kudu_127.6.8.252.keytab.
16:19:52.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:52.691267 6179 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVGVFlJ/build/tsan/bin/kudu
16:19:52.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskVGVFlJ/build/tsan/bin/kudu
16:19:52.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-2/wal
16:19:52.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-2/data
16:19:52.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:19:52.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:19:52.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:19:52.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:19:52.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:19:52.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:19:52.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-2/logs
16:19:52.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-2/data/info.pb
16:19:52.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:19:52.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:19:52.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:19:52.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:19:52.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:19:52.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:19:52.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:19:52.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:19:52.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:19:52.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:19:52.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:19:52.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:19:52.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.8.252:39609
16:19:52.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.8.252
16:19:52.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:19:52.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/krb5kdc/kudu_127.6.8.252.keytab
16:19:52.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.6.8.252
16:19:52.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:19:52.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:19:52.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:19:52.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.8.212:33651
16:19:52.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:19:52.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:19:52.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:19:52.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:19:52.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.8.254:34321,127.6.8.253:41923,127.6.8.252:39609
16:19:52.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:19:52.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44069/jwks.json
16:19:52.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:19:52.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:52.698968 6374 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.8.253" port: 41923 } has no permanent_uuid. Determining permanent_uuid...
16:19:52.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 16:19:52 dist-test-slave-cbbz krb5kdc[6192](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786465192, etypes {rep=17 tkt=17 ses=17}, kudu/127.6.8.253@KRBTEST.COM for kudu/127.6.8.253@KRBTEST.COM
16:19:52.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:52.736802 6374 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.8.252" port: 39609 } has no permanent_uuid. Determining permanent_uuid...
16:19:52.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:52.739712 6315 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.8.252:39609: connect: Connection refused (error 111)
16:19:52.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:52.744434 6374 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.8.252:39609: Network error: Client connection negotiation failed: client connection to 127.6.8.252:39609: connect: Connection refused (error 111)
16:19:52.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:52.793934 6374 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.8.252" port: 39609 } attempt: 1
16:19:52.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:52.798491 6374 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.8.252:39609: Network error: Client connection negotiation failed: client connection to 127.6.8.252:39609: connect: Connection refused (error 111)
16:19:52.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:52.867033 6374 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.8.252" port: 39609 } attempt: 2
16:19:52.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:52.872656 6374 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.8.252:39609: Network error: Client connection negotiation failed: client connection to 127.6.8.252:39609: connect: Connection refused (error 111)
16:19:52.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:52.964422 6374 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.8.252" port: 39609 } attempt: 3
16:19:52.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:52.968669 6374 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.8.252:39609: Network error: Client connection negotiation failed: client connection to 127.6.8.252:39609: connect: Connection refused (error 111)
16:19:53.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:53.061393 6381 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:19:53.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:53.062024 6381 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:19:53.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:53.062848 6381 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:19:53.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:53.064453 6283 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.8.253" port: 41923 } attempt: 7
16:19:53.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 16:19:53 dist-test-slave-cbbz krb5kdc[6192](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786465190, etypes {rep=17 tkt=17 ses=17}, kudu/127.6.8.254@KRBTEST.COM for kudu/127.6.8.253@KRBTEST.COM
16:19:53.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:53.106329 6283 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.8.252" port: 39609 } has no permanent_uuid. Determining permanent_uuid...
16:19:53.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:53.110253 6381 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
16:19:53.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:53.110574 6381 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:19:53.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:53.111081 6381 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:19:53.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:53.111402 6381 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:19:53.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:53.111596 6381 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:19:53.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:53.111815 6381 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:19:53.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:53.112088 6374 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.8.252" port: 39609 } attempt: 4
16:19:53.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:53.112787 6381 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44069/jwks.json
16:19:53.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:53.114157 6283 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.8.252:39609: Network error: Client connection negotiation failed: client connection to 127.6.8.252:39609: connect: Connection refused (error 111)
16:19:53.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:53.116415 6374 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.8.252:39609: Network error: Client connection negotiation failed: client connection to 127.6.8.252:39609: connect: Connection refused (error 111)
16:19:53.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:53.152128 6381 master_runner.cc:386] Master server non-default flags:
16:19:53.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:19:53.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.8.212:33651
16:19:53.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:19:53.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:19:53.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-2/data
16:19:53.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-2/wal
16:19:53.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
16:19:53.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:19:53.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.8.254:34321,127.6.8.253:41923,127.6.8.252:39609
16:19:53.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/krb5kdc/kudu_127.6.8.252.keytab
16:19:53.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.6.8.252
16:19:53.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:19:53.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:19:53.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:19:53.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.8.252:39609
16:19:53.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:19:53.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:19:53.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:19:53.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44069/jwks.json
16:19:53.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:19:53.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:19:53.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:19:53.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:19:53.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:19:53.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-2/data/info.pb
16:19:53.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.8.252
16:19:53.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:19:53.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:19:53.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:19:53.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:19:53.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:19:53.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:19:53.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-2/logs
16:19:53.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:19:53.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:19:53.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:19:53.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:19:53.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
16:19:53.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
16:19:53.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:19:53.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 15:12:52 UTC on ccycloud.quasar-tzrfpd.root.comops.site
16:19:53.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4580143
16:19:53.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:19:53.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:53.153352 6381 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:19:53.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:53.154276 6381 file_cache.cc:493] Constructed file cache file cache with capacity 419430
16:19:53.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:53.173087 6283 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.8.252" port: 39609 } attempt: 1
16:19:53.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:53.174621 6395 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:53.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:53.178668 6283 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.8.252:39609: Network error: Client connection negotiation failed: client connection to 127.6.8.252:39609: connect: Connection refused (error 111)
16:19:53.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:53.260114 6283 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.8.252" port: 39609 } attempt: 2
16:19:53.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:53.265621 6283 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.8.252:39609: Network error: Client connection negotiation failed: client connection to 127.6.8.252:39609: connect: Connection refused (error 111)
16:19:53.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:53.351208 6283 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.8.252" port: 39609 } attempt: 3
16:19:53.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:53.356788 6283 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.8.252:39609: Network error: Client connection negotiation failed: client connection to 127.6.8.252:39609: connect: Connection refused (error 111)
16:19:53.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:53.415843 6374 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.8.252" port: 39609 } attempt: 5
16:19:53.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:53.421214 6374 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.8.252:39609: Network error: Client connection negotiation failed: client connection to 127.6.8.252:39609: connect: Connection refused (error 111)
16:19:53.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:53.497265 6283 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.8.252" port: 39609 } attempt: 4
16:19:53.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:53.502784 6283 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.8.252:39609: Network error: Client connection negotiation failed: client connection to 127.6.8.252:39609: connect: Connection refused (error 111)
16:19:53.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:53.786247 6283 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.8.252" port: 39609 } attempt: 5
16:19:53.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:53.792245 6283 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.8.252:39609: Network error: Client connection negotiation failed: client connection to 127.6.8.252:39609: connect: Connection refused (error 111)
16:19:53.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:53.968677 6374 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.8.252" port: 39609 } attempt: 6
16:19:53.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:53.978107 6374 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.8.252:39609: Network error: Client connection negotiation failed: client connection to 127.6.8.252:39609: connect: Connection refused (error 111)
16:19:54.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:54.344820 6283 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.8.252" port: 39609 } attempt: 6
16:19:54.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:54.352941 6283 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.8.252:39609: Network error: Client connection negotiation failed: client connection to 127.6.8.252:39609: connect: Connection refused (error 111)
16:19:54.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:53.176421 6391 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:54.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:53.177172 6392 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:54.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:53.178205 6381 server_base.cc:1034] running on GCE node
16:19:54.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:54.490876 6381 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:19:54.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:54.493956 6381 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:19:54.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:54.495297 6381 hybrid_clock.cc:648] HybridClock initialized: now 1786465194495271 us; error 50 us; skew 500 ppm
16:19:54.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:54.496093 6381 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:19:54.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 16:19:54 dist-test-slave-cbbz krb5kdc[6192](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786465194, etypes {rep=17 tkt=17 ses=17}, kudu/127.6.8.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:19:54.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:54.502877 6381 init.cc:377] Logged in from keytab as kudu/127.6.8.252@KRBTEST.COM (short username kudu)
16:19:54.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:54.508528 6381 webserver.cc:460] Webserver started at http://127.6.8.252:37247/ using document root <none> and password file <none>
16:19:54.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:54.509379 6381 fs_manager.cc:359] Metadata directory not provided
16:19:54.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:54.509557 6381 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:19:54.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:54.509946 6381 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
16:19:54.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:54.514899 6381 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-2/data/instance:
16:19:54.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cb3c6ef4d12e40779471298ca16ad1fa"
16:19:54.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 16:19:54 on dist-test-slave-cbbz"
16:19:54.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:54.516021 6381 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-2/wal/instance:
16:19:54.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cb3c6ef4d12e40779471298ca16ad1fa"
16:19:54.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 16:19:54 on dist-test-slave-cbbz"
16:19:54.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:54.524602 6381 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.006s sys 0.001s
16:19:54.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:54.530524 6410 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:19:54.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:54.531764 6381 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.001s
16:19:54.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:54.532191 6381 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-2/data,/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-2/wal
16:19:54.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cb3c6ef4d12e40779471298ca16ad1fa"
16:19:54.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 16:19:54 on dist-test-slave-cbbz"
16:19:54.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:54.532661 6381 fs_report.cc:383] FS layout report
16:19:54.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:19:54.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-2/wal
16:19:54.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-2/wal
16:19:54.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-2/data/data
16:19:54.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:19:54.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:19:54.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:19:54.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:19:54.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:19:54.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:19:54.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:19:54.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:19:54.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:19:54.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:19:54.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:19:54.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:54.575577 6381 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
16:19:54.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:54.581429 6381 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:19:54.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:54.581980 6381 kserver.cc:163] Server-wide thread pool size limit: 3276
16:19:54.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:54.671120 6381 rpc_server.cc:307] RPC server started. Bound to: 127.6.8.252:39609
16:19:54.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:54.671263 6471 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.8.252:39609 every 8 connection(s)
16:19:54.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:54.673918 6381 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-2/data/info.pb
16:19:54.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:54.674926 6179 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVGVFlJ/build/tsan/bin/kudu as pid 6381
16:19:54.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:54.675493 6179 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-2/wal/instance
16:19:54.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:54.684794 6472 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
16:19:54.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:54.702507 6472 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.8.254" port: 34321 } has no permanent_uuid. Determining permanent_uuid...
16:19:54.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 16:19:54 dist-test-slave-cbbz krb5kdc[6192](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786465188, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.6.8.252@KRBTEST.COM
16:19:54.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 16:19:54 dist-test-slave-cbbz krb5kdc[6192](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786465194, etypes {rep=17 tkt=17 ses=17}, kudu/127.6.8.252@KRBTEST.COM for kudu/127.6.8.254@KRBTEST.COM
16:19:54.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:54.735275 6472 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.8.253" port: 41923 } has no permanent_uuid. Determining permanent_uuid...
16:19:54.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 16:19:54 dist-test-slave-cbbz krb5kdc[6192](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786465188, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.6.8.253@KRBTEST.COM
16:19:54.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 16:19:54 dist-test-slave-cbbz krb5kdc[6192](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786465194, etypes {rep=17 tkt=17 ses=17}, kudu/127.6.8.252@KRBTEST.COM for kudu/127.6.8.253@KRBTEST.COM
16:19:54.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:54.760974 6472 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.8.252" port: 39609 } has no permanent_uuid. Determining permanent_uuid...
16:19:54.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 16:19:54 dist-test-slave-cbbz krb5kdc[6192](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786465188, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.6.8.254@KRBTEST.COM
16:19:54.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 16:19:54 dist-test-slave-cbbz krb5kdc[6192](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786465194, etypes {rep=17 tkt=17 ses=17}, kudu/127.6.8.252@KRBTEST.COM for kudu/127.6.8.252@KRBTEST.COM
16:19:54.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:54.786661 6179 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
16:19:54.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:54.790858 6472 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa: Bootstrap starting.
16:19:54.794 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:19:54.795 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
16:19:54.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:54.796509 6472 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa: Neither blocks nor log segments found. Creating new log.
16:19:54.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:54.798487 6472 log.cc:824] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa: Log is configured to *not* fsync() on all Append() calls
16:19:54.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:54.803992 6472 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa: No bootstrap required, opened a new log
16:19:54.806 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
16:19:54.808 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
16:19:54.810 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.6.8.254"
port: 34321
}
bound_http_address {
host: "127.6.8.254"
port: 42747
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.6.8.253"
port: 41923
}
bound_http_address {
host: "127.6.8.253"
port: 41921
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.6.8.252"
port: 39609
}
bound_http_address {
host: "127.6.8.252"
port: 37247
}
}
}
16:19:54.810 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
16:19:54.810 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:19:54.811 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
16:19:54.812 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
16:19:54.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:54.834028 6472 raft_consensus.cc:348] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9636eb75872e49cb90a026bad879c251" member_type: VOTER last_known_addr { host: "127.6.8.254" port: 34321 } } peers { permanent_uuid: "a297c0bf389d4a5ca444bd373be38e48" member_type: VOTER last_known_addr { host: "127.6.8.253" port: 41923 } } peers { permanent_uuid: "cb3c6ef4d12e40779471298ca16ad1fa" member_type: VOTER last_known_addr { host: "127.6.8.252" port: 39609 } }
16:19:54.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:54.834899 6472 raft_consensus.cc:374] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
16:19:54.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:54.835202 6472 raft_consensus.cc:729] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: cb3c6ef4d12e40779471298ca16ad1fa, State: Initialized, Role: FOLLOWER
16:19:54.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:54.836186 6472 consensus_queue.cc:260] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9636eb75872e49cb90a026bad879c251" member_type: VOTER last_known_addr { host: "127.6.8.254" port: 34321 } } peers { permanent_uuid: "a297c0bf389d4a5ca444bd373be38e48" member_type: VOTER last_known_addr { host: "127.6.8.253" port: 41923 } } peers { permanent_uuid: "cb3c6ef4d12e40779471298ca16ad1fa" member_type: VOTER last_known_addr { host: "127.6.8.252" port: 39609 } }
16:19:54.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:54.839591 6483 sys_catalog.cc:455] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9636eb75872e49cb90a026bad879c251" member_type: VOTER last_known_addr { host: "127.6.8.254" port: 34321 } } peers { permanent_uuid: "a297c0bf389d4a5ca444bd373be38e48" member_type: VOTER last_known_addr { host: "127.6.8.253" port: 41923 } } peers { permanent_uuid: "cb3c6ef4d12e40779471298ca16ad1fa" member_type: VOTER last_known_addr { host: "127.6.8.252" port: 39609 } } }
16:19:54.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:54.840597 6483 sys_catalog.cc:458] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa [sys.catalog]: This master's current role is: FOLLOWER
16:19:54.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 16:19:54 dist-test-slave-cbbz krb5kdc[6192](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786465194, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:19:54.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COMI20260811 16:19:54.841166 6472 sys_catalog.cc:564] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa [sys.catalog]: configured and running, proceeding with master startup.
16:19:54.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) :
16:19:54.847 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:19:54.850 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/krb5kdc/krb5cc
16:19:54.858 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
16:19:54.859 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/krb5kdc/krb5cc
16:19:54.861 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
16:19:54.866 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
16:19:54.868 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
16:19:54.868 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
16:19:54.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:54.902518 6496 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:19:54.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:54.902909 6496 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:19:54.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 16:19:54 dist-test-slave-cbbz krb5kdc[6192](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786465194, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:19:54.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
16:19:54.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:54.882982 6381 master_runner.cc:186] Error getting master registration for 127.6.8.254:34321: OK, instance_id { permanent_uuid: "9636eb75872e49cb90a026bad879c251" instance_seqno: 1786465190643761 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
16:19:54.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:54.915503 6381 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:19:54.919 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:19:54.919 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/krb5kdc/krb5cc
16:19:54.920 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
16:19:54.921 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.6.8.252:39609
16:19:54.923 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
16:19:54.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:54.924331 6179 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVGVFlJ/build/tsan/bin/kudu with pid 6381
16:19:54.945 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:19:54.946 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.6.8.254:34321
16:19:54.946 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
16:19:54.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:54.946867 6179 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVGVFlJ/build/tsan/bin/kudu with pid 6205
16:19:54.971 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:19:54.971 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.6.8.253:41923
16:19:54.971 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
16:19:54.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:54.971807 6179 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVGVFlJ/build/tsan/bin/kudu with pid 6289
16:19:55.000 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:19:55.001 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.6.8.252:39609
16:19:55.002 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
16:19:55.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:55.003983 6179 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVGVFlJ/build/tsan/bin/kudu
16:19:55.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskVGVFlJ/build/tsan/bin/kudu
16:19:55.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-2/wal
16:19:55.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-2/data
16:19:55.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:19:55.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:19:55.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:19:55.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:19:55.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:19:55.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:19:55.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-2/logs
16:19:55.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-2/data/info.pb
16:19:55.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:19:55.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:19:55.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:19:55.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:19:55.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:19:55.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:19:55.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:19:55.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:19:55.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:19:55.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:19:55.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:19:55.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:19:55.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.8.252:39609
16:19:55.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.8.252
16:19:55.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37247
16:19:55.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/krb5kdc/kudu_127.6.8.252.keytab
16:19:55.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.6.8.252
16:19:55.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:19:55.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:19:55.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:19:55.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.8.212:33651
16:19:55.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:19:55.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:19:55.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:19:55.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:19:55.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.8.254:34321,127.6.8.253:41923,127.6.8.252:39609
16:19:55.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:19:55.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44069/jwks.json
16:19:55.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:19:55.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:55.328558 6499 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:19:55.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:55.329295 6499 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:19:55.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:55.330129 6499 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:19:55.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:55.362567 6499 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
16:19:55.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:55.362792 6499 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:19:55.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:55.363049 6499 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:19:55.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:55.363245 6499 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:19:55.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:55.363445 6499 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:19:55.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:55.363631 6499 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:19:55.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:55.363758 6499 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44069/jwks.json
16:19:55.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:55.400290 6499 master_runner.cc:386] Master server non-default flags:
16:19:55.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:19:55.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.8.212:33651
16:19:55.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:19:55.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:19:55.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-2/data
16:19:55.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-2/wal
16:19:55.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
16:19:55.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:19:55.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.8.254:34321,127.6.8.253:41923,127.6.8.252:39609
16:19:55.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/krb5kdc/kudu_127.6.8.252.keytab
16:19:55.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.6.8.252
16:19:55.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:19:55.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:19:55.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:19:55.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.8.252:39609
16:19:55.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:19:55.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:19:55.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:19:55.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44069/jwks.json
16:19:55.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:19:55.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:19:55.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:19:55.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:19:55.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:19:55.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-2/data/info.pb
16:19:55.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.8.252
16:19:55.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37247
16:19:55.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:19:55.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:19:55.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:19:55.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:19:55.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:19:55.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-2/logs
16:19:55.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:19:55.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:19:55.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:19:55.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:19:55.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
16:19:55.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
16:19:55.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:19:55.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 15:12:52 UTC on ccycloud.quasar-tzrfpd.root.comops.site
16:19:55.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4580143
16:19:55.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:19:55.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:55.401548 6499 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:19:55.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:55.402510 6499 file_cache.cc:493] Constructed file cache file cache with capacity 419430
16:19:55.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:55.430727 6505 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:56.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:55.431250 6508 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:56.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:55.432715 6506 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:56.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:56.643998 6499 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:19:56.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:56.644659 6507 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
16:19:56.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:56.650756 6499 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:19:56.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:56.653797 6499 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:19:56.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:56.655241 6499 hybrid_clock.cc:648] HybridClock initialized: now 1786465196655196 us; error 51 us; skew 500 ppm
16:19:56.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:56.656242 6499 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:19:56.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 16:19:56 dist-test-slave-cbbz krb5kdc[6192](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786465196, etypes {rep=17 tkt=17 ses=17}, kudu/127.6.8.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:19:56.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:56.667361 6499 init.cc:377] Logged in from keytab as kudu/127.6.8.252@KRBTEST.COM (short username kudu)
16:19:56.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:56.673055 6499 webserver.cc:460] Webserver started at http://127.6.8.252:37247/ using document root <none> and password file <none>
16:19:56.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:56.674278 6499 fs_manager.cc:359] Metadata directory not provided
16:19:56.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:56.674578 6499 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:19:56.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:56.682889 6499 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.005s sys 0.000s
16:19:56.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:56.687430 6520 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:19:56.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:56.688547 6499 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
16:19:56.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:56.688959 6499 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-2/data,/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-2/wal
16:19:56.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cb3c6ef4d12e40779471298ca16ad1fa"
16:19:56.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 16:19:54 on dist-test-slave-cbbz"
16:19:56.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:56.691361 6499 fs_report.cc:383] FS layout report
16:19:56.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:19:56.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-2/wal
16:19:56.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-2/wal
16:19:56.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-2/data/data
16:19:56.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:19:56.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:19:56.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:19:56.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:19:56.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:19:56.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:19:56.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:19:56.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:19:56.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:19:56.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:19:56.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:19:56.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:56.738226 6499 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
16:19:56.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:56.744645 6499 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:19:56.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:56.745102 6499 kserver.cc:163] Server-wide thread pool size limit: 3276
16:19:56.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:56.816642 6499 rpc_server.cc:307] RPC server started. Bound to: 127.6.8.252:39609
16:19:56.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:56.816720 6582 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.8.252:39609 every 8 connection(s)
16:19:56.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:56.819550 6499 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-2/data/info.pb
16:19:56.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:56.828890 6179 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVGVFlJ/build/tsan/bin/kudu as pid 6499
16:19:56.829 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:19:56.829 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.6.8.254:34321
16:19:56.830 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
16:19:56.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:56.830211 6583 sys_catalog.cc:263] Verifying existing consensus state
16:19:56.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:56.831626 6179 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVGVFlJ/build/tsan/bin/kudu
16:19:56.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskVGVFlJ/build/tsan/bin/kudu
16:19:56.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-0/wal
16:19:56.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-0/data
16:19:56.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:19:56.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:19:56.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:19:56.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:19:56.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:19:56.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:19:56.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-0/logs
16:19:56.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-0/data/info.pb
16:19:56.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:19:56.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:19:56.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:19:56.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:19:56.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:19:56.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:19:56.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:19:56.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:19:56.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:19:56.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:19:56.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:19:56.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:19:56.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.8.254:34321
16:19:56.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.8.254
16:19:56.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42747
16:19:56.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/krb5kdc/kudu_127.6.8.254.keytab
16:19:56.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.6.8.254
16:19:56.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:19:56.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:19:56.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:19:56.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.8.212:33651
16:19:56.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:19:56.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:19:56.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:19:56.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:19:56.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.8.254:34321,127.6.8.253:41923,127.6.8.252:39609
16:19:56.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:19:56.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44069/jwks.json
16:19:56.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:19:56.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:56.837049 6583 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa: Bootstrap starting.
16:19:56.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:56.883397 6583 log.cc:824] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa: Log is configured to *not* fsync() on all Append() calls
16:19:56.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:56.888563 6583 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
16:19:56.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:56.889367 6583 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa: Bootstrap complete.
16:19:56.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:56.914587 6583 raft_consensus.cc:348] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9636eb75872e49cb90a026bad879c251" member_type: VOTER last_known_addr { host: "127.6.8.254" port: 34321 } } peers { permanent_uuid: "a297c0bf389d4a5ca444bd373be38e48" member_type: VOTER last_known_addr { host: "127.6.8.253" port: 41923 } } peers { permanent_uuid: "cb3c6ef4d12e40779471298ca16ad1fa" member_type: VOTER last_known_addr { host: "127.6.8.252" port: 39609 } }
16:19:56.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:56.917043 6583 raft_consensus.cc:374] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
16:19:56.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:56.917333 6583 raft_consensus.cc:729] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: cb3c6ef4d12e40779471298ca16ad1fa, State: Initialized, Role: FOLLOWER
16:19:56.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:56.918186 6583 consensus_queue.cc:260] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9636eb75872e49cb90a026bad879c251" member_type: VOTER last_known_addr { host: "127.6.8.254" port: 34321 } } peers { permanent_uuid: "a297c0bf389d4a5ca444bd373be38e48" member_type: VOTER last_known_addr { host: "127.6.8.253" port: 41923 } } peers { permanent_uuid: "cb3c6ef4d12e40779471298ca16ad1fa" member_type: VOTER last_known_addr { host: "127.6.8.252" port: 39609 } }
16:19:56.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:56.920885 6588 sys_catalog.cc:455] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9636eb75872e49cb90a026bad879c251" member_type: VOTER last_known_addr { host: "127.6.8.254" port: 34321 } } peers { permanent_uuid: "a297c0bf389d4a5ca444bd373be38e48" member_type: VOTER last_known_addr { host: "127.6.8.253" port: 41923 } } peers { permanent_uuid: "cb3c6ef4d12e40779471298ca16ad1fa" member_type: VOTER last_known_addr { host: "127.6.8.252" port: 39609 } } }
16:19:56.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:56.921733 6588 sys_catalog.cc:458] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa [sys.catalog]: This master's current role is: FOLLOWER
16:19:56.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:56.922106 6583 sys_catalog.cc:564] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa [sys.catalog]: configured and running, proceeding with master startup.
16:19:56.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:56.953558 6601 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:19:56.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:56.954171 6601 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:19:56.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:56.949891 6499 master_runner.cc:186] Error getting master registration for 127.6.8.254:34321: Network error: Client connection negotiation failed: client connection to 127.6.8.254:34321: connect: Connection refused (error 111),
16:19:56.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:56.983052 6499 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:19:56.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:56.990775 6588 raft_consensus.cc:482] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
16:19:56.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:56.991376 6588 raft_consensus.cc:504] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9636eb75872e49cb90a026bad879c251" member_type: VOTER last_known_addr { host: "127.6.8.254" port: 34321 } } peers { permanent_uuid: "a297c0bf389d4a5ca444bd373be38e48" member_type: VOTER last_known_addr { host: "127.6.8.253" port: 41923 } } peers { permanent_uuid: "cb3c6ef4d12e40779471298ca16ad1fa" member_type: VOTER last_known_addr { host: "127.6.8.252" port: 39609 } }
16:19:56.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:56.995702 6588 leader_election.cc:288] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 9636eb75872e49cb90a026bad879c251 (127.6.8.254:34321), a297c0bf389d4a5ca444bd373be38e48 (127.6.8.253:41923)
16:19:57.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:57.000789 6527 tablet.cc:2369] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa: Can't schedule compaction. Clean time has not been advanced past its initial value.
16:19:57.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:57.007571 6523 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.8.254:34321: connect: Connection refused (error 111)
16:19:57.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:57.025328 6523 leader_election.cc:334] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 9636eb75872e49cb90a026bad879c251 (127.6.8.254:34321): Network error: Client connection negotiation failed: client connection to 127.6.8.254:34321: connect: Connection refused (error 111)
16:19:57.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:57.027732 6522 leader_election.cc:334] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a297c0bf389d4a5ca444bd373be38e48 (127.6.8.253:41923): Network error: Client connection negotiation failed: client connection to 127.6.8.253:41923: connect: Connection refused (error 111)
16:19:57.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:57.028182 6522 leader_election.cc:302] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: cb3c6ef4d12e40779471298ca16ad1fa; no voters: 9636eb75872e49cb90a026bad879c251, a297c0bf389d4a5ca444bd373be38e48
16:19:57.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:57.029440 6588 raft_consensus.cc:2726] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
16:19:57.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:57.217339 6585 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:19:57.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:57.217908 6585 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:19:57.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:57.218483 6585 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:19:57.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:57.256032 6585 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
16:19:57.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:57.256322 6585 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:19:57.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:57.256589 6585 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:19:57.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:57.256790 6585 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:19:57.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:57.256974 6585 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:19:57.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:57.257156 6585 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:19:57.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:57.257297 6585 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44069/jwks.json
16:19:57.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:57.293057 6585 master_runner.cc:386] Master server non-default flags:
16:19:57.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:19:57.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.8.212:33651
16:19:57.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:19:57.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:19:57.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-0/data
16:19:57.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-0/wal
16:19:57.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
16:19:57.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:19:57.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.8.254:34321,127.6.8.253:41923,127.6.8.252:39609
16:19:57.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/krb5kdc/kudu_127.6.8.254.keytab
16:19:57.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.6.8.254
16:19:57.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:19:57.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:19:57.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:19:57.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.8.254:34321
16:19:57.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:19:57.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:19:57.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:19:57.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44069/jwks.json
16:19:57.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:19:57.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:19:57.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:19:57.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:19:57.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:19:57.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-0/data/info.pb
16:19:57.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.8.254
16:19:57.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42747
16:19:57.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:19:57.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:19:57.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:19:57.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:19:57.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:19:57.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-0/logs
16:19:57.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:19:57.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:19:57.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:19:57.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:19:57.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
16:19:57.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
16:19:57.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:19:57.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 15:12:52 UTC on ccycloud.quasar-tzrfpd.root.comops.site
16:19:57.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4580143
16:19:57.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:19:57.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:57.294328 6585 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:19:57.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:57.295650 6585 file_cache.cc:493] Constructed file cache file cache with capacity 419430
16:19:57.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:57.316434 6616 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:57.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:57.958884 6601 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:19:57.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:57.959225 6601 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:19:58.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:58.567816 6623 raft_consensus.cc:482] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
16:19:58.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:58.568379 6623 raft_consensus.cc:504] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9636eb75872e49cb90a026bad879c251" member_type: VOTER last_known_addr { host: "127.6.8.254" port: 34321 } } peers { permanent_uuid: "a297c0bf389d4a5ca444bd373be38e48" member_type: VOTER last_known_addr { host: "127.6.8.253" port: 41923 } } peers { permanent_uuid: "cb3c6ef4d12e40779471298ca16ad1fa" member_type: VOTER last_known_addr { host: "127.6.8.252" port: 39609 } }
16:19:58.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:58.570370 6623 leader_election.cc:288] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 9636eb75872e49cb90a026bad879c251 (127.6.8.254:34321), a297c0bf389d4a5ca444bd373be38e48 (127.6.8.253:41923)
16:19:58.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:58.585426 6523 leader_election.cc:334] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 9636eb75872e49cb90a026bad879c251 (127.6.8.254:34321): Network error: Client connection negotiation failed: client connection to 127.6.8.254:34321: connect: Connection refused (error 111)
16:19:58.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:58.586098 6522 leader_election.cc:334] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a297c0bf389d4a5ca444bd373be38e48 (127.6.8.253:41923): Network error: Client connection negotiation failed: client connection to 127.6.8.253:41923: connect: Connection refused (error 111)
16:19:58.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:58.586480 6522 leader_election.cc:302] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: cb3c6ef4d12e40779471298ca16ad1fa; no voters: 9636eb75872e49cb90a026bad879c251, a297c0bf389d4a5ca444bd373be38e48
16:19:58.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:58.587368 6623 raft_consensus.cc:2726] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
16:19:58.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:57.319130 6619 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:58.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:57.316479 6615 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:58.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:58.599830 6618 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
16:19:58.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:58.600224 6585 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:19:58.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:58.603899 6585 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:19:58.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:58.606704 6585 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:19:58.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:58.608188 6585 hybrid_clock.cc:648] HybridClock initialized: now 1786465198608126 us; error 54 us; skew 500 ppm
16:19:58.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:58.609118 6585 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:19:58.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 16:19:58 dist-test-slave-cbbz krb5kdc[6192](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786465198, etypes {rep=17 tkt=17 ses=17}, kudu/127.6.8.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:19:58.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:58.615244 6585 init.cc:377] Logged in from keytab as kudu/127.6.8.254@KRBTEST.COM (short username kudu)
16:19:58.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:58.621083 6585 webserver.cc:460] Webserver started at http://127.6.8.254:42747/ using document root <none> and password file <none>
16:19:58.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:58.622025 6585 fs_manager.cc:359] Metadata directory not provided
16:19:58.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:58.622330 6585 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:19:58.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:58.631150 6585 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.004s sys 0.002s
16:19:58.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:58.636298 6636 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:19:58.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:58.637583 6585 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
16:19:58.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:58.637943 6585 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-0/data,/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-0/wal
16:19:58.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9636eb75872e49cb90a026bad879c251"
16:19:58.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 16:19:50 on dist-test-slave-cbbz"
16:19:58.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:58.641587 6585 fs_report.cc:383] FS layout report
16:19:58.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:19:58.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-0/wal
16:19:58.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-0/wal
16:19:58.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-0/data/data
16:19:58.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:19:58.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:19:58.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:19:58.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:19:58.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:19:58.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:19:58.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:19:58.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:19:58.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:19:58.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:19:58.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:19:58.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:58.692049 6585 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
16:19:58.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:58.699946 6585 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:19:58.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:58.700572 6585 kserver.cc:163] Server-wide thread pool size limit: 3276
16:19:58.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:58.786974 6585 rpc_server.cc:307] RPC server started. Bound to: 127.6.8.254:34321
16:19:58.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:58.787083 6693 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.8.254:34321 every 8 connection(s)
16:19:58.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:58.790962 6585 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-0/data/info.pb
16:19:58.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:58.793499 6179 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVGVFlJ/build/tsan/bin/kudu as pid 6585
16:19:58.795 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:19:58.795 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.6.8.253:41923
16:19:58.795 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
16:19:58.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:58.798405 6179 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVGVFlJ/build/tsan/bin/kudu
16:19:58.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskVGVFlJ/build/tsan/bin/kudu
16:19:58.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-1/wal
16:19:58.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-1/data
16:19:58.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:19:58.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:19:58.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:19:58.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:19:58.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:19:58.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:19:58.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-1/logs
16:19:58.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-1/data/info.pb
16:19:58.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:19:58.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:19:58.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:19:58.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:19:58.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:19:58.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:19:58.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:19:58.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:19:58.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:19:58.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:19:58.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:19:58.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:19:58.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.8.253:41923
16:19:58.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.8.253
16:19:58.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41921
16:19:58.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/krb5kdc/kudu_127.6.8.253.keytab
16:19:58.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.6.8.253
16:19:58.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:19:58.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:19:58.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:19:58.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.8.212:33651
16:19:58.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:19:58.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:19:58.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:19:58.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:19:58.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.8.254:34321,127.6.8.253:41923,127.6.8.252:39609
16:19:58.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:19:58.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44069/jwks.json
16:19:58.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:19:58.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:58.812965 6694 sys_catalog.cc:263] Verifying existing consensus state
16:19:58.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260811 16:19:58.814177 6694 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
16:19:58.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:58.814759 6585 master.cc:537] Master@127.6.8.254:34321 shutting down...
16:19:58.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:58.907426 6585 master.cc:552] Master@127.6.8.254:34321 shutdown complete.
16:19:58.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
16:19:58.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:58.962319 6601 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:19:58.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:58.962574 6601 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:19:59.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:59.163584 6696 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:19:59.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:59.164325 6696 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:19:59.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:59.165043 6696 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:19:59.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:59.196521 6696 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
16:19:59.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:59.196805 6696 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:19:59.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:59.197189 6696 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:19:59.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:59.197520 6696 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:19:59.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:59.197831 6696 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:19:59.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:59.198153 6696 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:19:59.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:59.198386 6696 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44069/jwks.json
16:19:59.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:59.238754 6696 master_runner.cc:386] Master server non-default flags:
16:19:59.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:19:59.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.8.212:33651
16:19:59.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:19:59.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:19:59.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-1/data
16:19:59.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-1/wal
16:19:59.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
16:19:59.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:19:59.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.8.254:34321,127.6.8.253:41923,127.6.8.252:39609
16:19:59.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/krb5kdc/kudu_127.6.8.253.keytab
16:19:59.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.6.8.253
16:19:59.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:19:59.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:19:59.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:19:59.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.8.253:41923
16:19:59.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:19:59.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:19:59.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:19:59.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44069/jwks.json
16:19:59.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:19:59.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:19:59.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:19:59.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:19:59.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:19:59.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-1/data/info.pb
16:19:59.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.8.253
16:19:59.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41921
16:19:59.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:19:59.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:19:59.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:19:59.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:19:59.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:19:59.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-1/logs
16:19:59.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:19:59.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:19:59.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:19:59.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:19:59.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
16:19:59.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
16:19:59.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:19:59.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 15:12:52 UTC on ccycloud.quasar-tzrfpd.root.comops.site
16:19:59.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4580143
16:19:59.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:19:59.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:59.240285 6696 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:19:59.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:59.241602 6696 file_cache.cc:493] Constructed file cache file cache with capacity 419430
16:19:59.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:59.262329 6703 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:59.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:59.966056 6601 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:19:59.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:59.966425 6601 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:20:00.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:59.262758 6704 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:20:00.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:19:59.264345 6696 server_base.cc:1034] running on GCE node
16:20:00.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:19:59.264886 6706 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:20:00.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:00.526533 6696 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:20:00.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:00.529278 6696 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:20:00.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:00.530704 6696 hybrid_clock.cc:648] HybridClock initialized: now 1786465200530671 us; error 41 us; skew 500 ppm
16:20:00.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:00.531737 6696 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:20:00.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 16:20:00 dist-test-slave-cbbz krb5kdc[6192](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786465200, etypes {rep=17 tkt=17 ses=17}, kudu/127.6.8.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:20:00.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:00.537789 6696 init.cc:377] Logged in from keytab as kudu/127.6.8.253@KRBTEST.COM (short username kudu)
16:20:00.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:00.542651 6696 webserver.cc:460] Webserver started at http://127.6.8.253:41921/ using document root <none> and password file <none>
16:20:00.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:00.543781 6696 fs_manager.cc:359] Metadata directory not provided
16:20:00.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:00.544070 6696 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:20:00.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:00.553188 6696 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.004s sys 0.004s
16:20:00.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:00.558354 6718 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:20:00.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:00.559549 6696 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
16:20:00.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:00.559911 6696 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-1/data,/tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-1/wal
16:20:00.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a297c0bf389d4a5ca444bd373be38e48"
16:20:00.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 16:19:52 on dist-test-slave-cbbz"
16:20:00.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:00.562847 6696 fs_report.cc:383] FS layout report
16:20:00.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:20:00.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-1/wal
16:20:00.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-1/wal
16:20:00.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-1/data/data
16:20:00.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:20:00.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:20:00.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:20:00.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:20:00.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:20:00.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:20:00.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:20:00.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:20:00.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:20:00.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:20:00.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:20:00.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:00.610323 6696 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
16:20:00.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:00.616595 6696 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:20:00.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:00.617092 6696 kserver.cc:163] Server-wide thread pool size limit: 3276
16:20:00.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:00.710937 6696 rpc_server.cc:307] RPC server started. Bound to: 127.6.8.253:41923
16:20:00.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:00.711074 6777 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.8.253:41923 every 8 connection(s)
16:20:00.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:00.714311 6696 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-1/data/info.pb
16:20:00.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:00.724195 6179 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVGVFlJ/build/tsan/bin/kudu as pid 6696
16:20:00.724 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:20:00.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:00.725747 6778 sys_catalog.cc:263] Verifying existing consensus state
16:20:00.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260811 16:20:00.726449 6778 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
16:20:00.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:00.726830 6696 master.cc:537] Master@127.6.8.253:41923 shutting down...
16:20:00.786 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.6.8.252:39609(127.6.8.252:39609)] connecting to peer
16:20:00.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:00.798593 6696 master.cc:552] Master@127.6.8.253:41923 shutdown complete.
16:20:00.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskVGVFlJ/test-tmp/mini-kudu-cluster7476366384262101532/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
16:20:00.851 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.6.8.254:34321(127.6.8.254:34321)] connecting to peer
16:20:00.860 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.6.8.253:41923(127.6.8.253:41923)] connecting to peer
16:20:00.897 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.6.8.252:39609(127.6.8.252:39609)] Successfully connected to peer
16:20:00.900 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:00.901 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.8.254:34321(127.6.8.254:34321)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:00.902 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.8.254:34321: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:00.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:00.969547 6601 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:20:00.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:00.969841 6601 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:20:00.975 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:00.975 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.8.253:41923(127.6.8.253:41923)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:00.976 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.8.253:41923: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:00.991 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
16:20:01.002 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.6.8.252
16:20:01.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 16:20:01 dist-test-slave-cbbz krb5kdc[6192](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786465194, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.6.8.252@KRBTEST.COM
16:20:01.235 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x09328e2a, L:/127.0.0.1:53506 - R:/127.6.8.252:39609] using SASL/GSSAPI
16:20:01.251 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923; will retry
16:20:01.274 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.8.254:34321(127.6.8.254:34321)] connecting to peer
16:20:01.277 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:01.277 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.8.253:41923(127.6.8.253:41923)] connecting to peer
16:20:01.278 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.8.254:34321(127.6.8.254:34321)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:01.279 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.8.254:34321: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:01.281 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:01.281 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.8.253:41923(127.6.8.253:41923)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:01.282 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.8.253:41923: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:01.282 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923; will retry
16:20:01.291 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.8.254:34321(127.6.8.254:34321)] connecting to peer
16:20:01.293 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.8.253:41923(127.6.8.253:41923)] connecting to peer
16:20:01.294 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:01.295 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.8.254:34321(127.6.8.254:34321)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:01.295 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.8.254:34321: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:01.296 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:01.296 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.8.253:41923(127.6.8.253:41923)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:01.297 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.8.253:41923: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:01.297 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923; will retry
16:20:01.312 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.8.254:34321(127.6.8.254:34321)] connecting to peer
16:20:01.313 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.8.253:41923(127.6.8.253:41923)] connecting to peer
16:20:01.316 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:01.316 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.8.254:34321(127.6.8.254:34321)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:01.317 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.8.254:34321: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:01.318 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:01.318 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.8.253:41923(127.6.8.253:41923)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:01.319 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.8.253:41923: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:01.319 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923; will retry
16:20:01.331 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.8.254:34321(127.6.8.254:34321)] connecting to peer
16:20:01.332 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.8.253:41923(127.6.8.253:41923)] connecting to peer
16:20:01.335 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:01.335 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.8.254:34321(127.6.8.254:34321)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:01.336 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.8.254:34321: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:01.337 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:01.337 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.8.253:41923(127.6.8.253:41923)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:01.338 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.8.253:41923: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:01.338 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923; will retry
16:20:01.372 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.8.254:34321(127.6.8.254:34321)] connecting to peer
16:20:01.374 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.8.253:41923(127.6.8.253:41923)] connecting to peer
16:20:01.375 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:01.375 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.8.254:34321(127.6.8.254:34321)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:01.377 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.8.254:34321: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:01.379 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:01.380 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.8.253:41923(127.6.8.253:41923)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:01.380 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.8.253:41923: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:01.381 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923; will retry
16:20:01.451 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.8.254:34321(127.6.8.254:34321)] connecting to peer
16:20:01.453 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.8.253:41923(127.6.8.253:41923)] connecting to peer
16:20:01.456 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:01.456 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.8.254:34321(127.6.8.254:34321)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:01.457 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.8.254:34321: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:01.458 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:01.458 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.8.253:41923(127.6.8.253:41923)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:01.459 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.8.253:41923: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:01.459 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923; will retry
16:20:01.551 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.8.254:34321(127.6.8.254:34321)] connecting to peer
16:20:01.553 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.8.253:41923(127.6.8.253:41923)] connecting to peer
16:20:01.555 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:01.555 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.8.254:34321(127.6.8.254:34321)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:01.556 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.8.254:34321: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:01.557 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:01.557 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.8.253:41923(127.6.8.253:41923)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:01.558 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.8.253:41923: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:01.558 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923; will retry
16:20:01.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:01.599786 6784 raft_consensus.cc:482] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
16:20:01.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:01.600234 6784 raft_consensus.cc:504] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9636eb75872e49cb90a026bad879c251" member_type: VOTER last_known_addr { host: "127.6.8.254" port: 34321 } } peers { permanent_uuid: "a297c0bf389d4a5ca444bd373be38e48" member_type: VOTER last_known_addr { host: "127.6.8.253" port: 41923 } } peers { permanent_uuid: "cb3c6ef4d12e40779471298ca16ad1fa" member_type: VOTER last_known_addr { host: "127.6.8.252" port: 39609 } }
16:20:01.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:01.602257 6784 leader_election.cc:288] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 9636eb75872e49cb90a026bad879c251 (127.6.8.254:34321), a297c0bf389d4a5ca444bd373be38e48 (127.6.8.253:41923)
16:20:01.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:01.609428 6523 leader_election.cc:334] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 9636eb75872e49cb90a026bad879c251 (127.6.8.254:34321): Network error: Client connection negotiation failed: client connection to 127.6.8.254:34321: connect: Connection refused (error 111)
16:20:01.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:01.609874 6522 leader_election.cc:334] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a297c0bf389d4a5ca444bd373be38e48 (127.6.8.253:41923): Network error: Client connection negotiation failed: client connection to 127.6.8.253:41923: connect: Connection refused (error 111)
16:20:01.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:01.610245 6522 leader_election.cc:302] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: cb3c6ef4d12e40779471298ca16ad1fa; no voters: 9636eb75872e49cb90a026bad879c251, a297c0bf389d4a5ca444bd373be38e48
16:20:01.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:01.610882 6784 raft_consensus.cc:2726] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
16:20:01.632 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.8.254:34321(127.6.8.254:34321)] connecting to peer
16:20:01.633 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.8.253:41923(127.6.8.253:41923)] connecting to peer
16:20:01.634 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:01.634 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.8.254:34321(127.6.8.254:34321)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:01.635 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.8.254:34321: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:01.637 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:01.637 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.8.253:41923(127.6.8.253:41923)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:01.638 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.8.253:41923: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:01.638 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923; will retry
16:20:01.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:01.972877 6601 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:20:01.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:01.973150 6601 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:20:01.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:01.985298 6499 master_runner.cc:186] Error getting master registration for 127.6.8.254:34321: Network error: Client connection negotiation failed: client connection to 127.6.8.254:34321: connect: Connection refused (error 111),
16:20:01.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:01.985708 6499 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:20:02.072 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.8.254:34321(127.6.8.254:34321)] connecting to peer
16:20:02.074 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.8.253:41923(127.6.8.253:41923)] connecting to peer
16:20:02.074 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:02.075 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.8.254:34321(127.6.8.254:34321)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:02.076 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.8.254:34321: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:02.077 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:02.078 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.8.253:41923(127.6.8.253:41923)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:02.078 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.8.253:41923: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:02.079 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923; will retry
16:20:02.332 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.8.254:34321(127.6.8.254:34321)] connecting to peer
16:20:02.333 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.8.253:41923(127.6.8.253:41923)] connecting to peer
16:20:02.335 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:02.335 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.8.254:34321(127.6.8.254:34321)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:02.335 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.8.254:34321: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:02.336 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:02.337 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.8.253:41923(127.6.8.253:41923)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:02.337 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.8.253:41923: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:02.337 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923; will retry
16:20:02.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:02.975956 6601 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:20:02.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:02.976297 6601 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:20:02.992 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.8.254:34321(127.6.8.254:34321)] connecting to peer
16:20:02.994 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:02.994 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.8.254:34321(127.6.8.254:34321)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:02.994 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.8.254:34321: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:02.995 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.8.253:41923(127.6.8.253:41923)] connecting to peer
16:20:02.997 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:02.997 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.8.253:41923(127.6.8.253:41923)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:02.997 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.8.253:41923: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:02.997 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923; will retry
16:20:03.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:03.979427 6601 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:20:03.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:03.979724 6601 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:20:04.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:04.982729 6601 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:20:04.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:04.983036 6601 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:20:05.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:05.644964 6796 raft_consensus.cc:482] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
16:20:05.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:05.645664 6796 raft_consensus.cc:504] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9636eb75872e49cb90a026bad879c251" member_type: VOTER last_known_addr { host: "127.6.8.254" port: 34321 } } peers { permanent_uuid: "a297c0bf389d4a5ca444bd373be38e48" member_type: VOTER last_known_addr { host: "127.6.8.253" port: 41923 } } peers { permanent_uuid: "cb3c6ef4d12e40779471298ca16ad1fa" member_type: VOTER last_known_addr { host: "127.6.8.252" port: 39609 } }
16:20:05.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:05.647939 6796 leader_election.cc:288] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 9636eb75872e49cb90a026bad879c251 (127.6.8.254:34321), a297c0bf389d4a5ca444bd373be38e48 (127.6.8.253:41923)
16:20:05.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:05.651170 6523 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.8.254:34321: connect: Connection refused (error 111) [suppressed 5 similar messages]
16:20:05.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:05.654510 6522 leader_election.cc:334] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a297c0bf389d4a5ca444bd373be38e48 (127.6.8.253:41923): Network error: Client connection negotiation failed: client connection to 127.6.8.253:41923: connect: Connection refused (error 111)
16:20:05.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:05.654994 6523 leader_election.cc:334] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 9636eb75872e49cb90a026bad879c251 (127.6.8.254:34321): Network error: Client connection negotiation failed: client connection to 127.6.8.254:34321: connect: Connection refused (error 111)
16:20:05.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:05.655385 6523 leader_election.cc:302] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: cb3c6ef4d12e40779471298ca16ad1fa; no voters: 9636eb75872e49cb90a026bad879c251, a297c0bf389d4a5ca444bd373be38e48
16:20:05.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:05.656098 6796 raft_consensus.cc:2726] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
16:20:05.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:05.987007 6601 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:20:05.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:05.987393 6601 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:20:06.652 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.8.254:34321(127.6.8.254:34321)] connecting to peer
16:20:06.654 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.8.253:41923(127.6.8.253:41923)] connecting to peer
16:20:06.656 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:06.656 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.8.254:34321(127.6.8.254:34321)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:06.656 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.8.254:34321: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:06.657 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:06.657 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.8.253:41923(127.6.8.253:41923)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:06.657 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.8.253:41923: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:06.657 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923; will retry
16:20:06.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 16:20:06 dist-test-slave-cbbz krb5kdc[6192](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786465206, etypes {rep=17 tkt=17 ses=17}, kudu/127.6.8.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:20:06.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:06.671461 6515 init.cc:295] Successfully reacquired a new kerberos TGT
16:20:06.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:06.989979 6499 master_runner.cc:186] Error getting master registration for 127.6.8.254:34321: Network error: Client connection negotiation failed: client connection to 127.6.8.254:34321: connect: Connection refused (error 111),
16:20:06.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:06.990597 6499 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:20:06.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:06.991667 6601 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:20:06.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:06.991987 6601 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:20:07.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:07.996261 6601 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:20:07.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:07.996618 6601 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:20:08.691 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.8.254:34321(127.6.8.254:34321)] connecting to peer
16:20:08.692 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.8.253:41923(127.6.8.253:41923)] connecting to peer
16:20:08.694 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:08.694 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:08.694 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.8.253:41923(127.6.8.253:41923)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:08.694 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.8.254:34321(127.6.8.254:34321)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:08.694 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.8.253:41923: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:08.695 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.8.254:34321: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:08.695 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923; will retry
16:20:09.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:09.000610 6601 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:20:09.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:09.000942 6601 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:20:09.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:09.082201 6005 debug-util.cc:398] Leaking SignalData structure 0x7b08000aff80 after lost signal to thread 5869
16:20:09.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:09.083201 6005 debug-util.cc:398] Leaking SignalData structure 0x7b0800042140 after lost signal to thread 6008
16:20:09.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:09.484000 6805 raft_consensus.cc:482] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
16:20:09.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:09.484488 6805 raft_consensus.cc:504] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9636eb75872e49cb90a026bad879c251" member_type: VOTER last_known_addr { host: "127.6.8.254" port: 34321 } } peers { permanent_uuid: "a297c0bf389d4a5ca444bd373be38e48" member_type: VOTER last_known_addr { host: "127.6.8.253" port: 41923 } } peers { permanent_uuid: "cb3c6ef4d12e40779471298ca16ad1fa" member_type: VOTER last_known_addr { host: "127.6.8.252" port: 39609 } }
16:20:09.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:09.486387 6805 leader_election.cc:288] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 9636eb75872e49cb90a026bad879c251 (127.6.8.254:34321), a297c0bf389d4a5ca444bd373be38e48 (127.6.8.253:41923)
16:20:09.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:09.491631 6523 leader_election.cc:334] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 9636eb75872e49cb90a026bad879c251 (127.6.8.254:34321): Network error: Client connection negotiation failed: client connection to 127.6.8.254:34321: connect: Connection refused (error 111)
16:20:09.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:09.492085 6522 leader_election.cc:334] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a297c0bf389d4a5ca444bd373be38e48 (127.6.8.253:41923): Network error: Client connection negotiation failed: client connection to 127.6.8.253:41923: connect: Connection refused (error 111)
16:20:09.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:09.492569 6522 leader_election.cc:302] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: cb3c6ef4d12e40779471298ca16ad1fa; no voters: 9636eb75872e49cb90a026bad879c251, a297c0bf389d4a5ca444bd373be38e48
16:20:09.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:09.493350 6805 raft_consensus.cc:2726] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
16:20:10.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:10.005280 6601 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:20:10.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:10.005640 6601 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:20:11.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:11.008800 6601 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:20:11.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:11.009053 6601 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:20:11.671 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.8.254:34321(127.6.8.254:34321)] connecting to peer
16:20:11.673 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.8.253:41923(127.6.8.253:41923)] connecting to peer
16:20:11.673 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:11.674 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.8.254:34321(127.6.8.254:34321)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:11.674 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.8.254:34321: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:11.676 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:11.676 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.8.253:41923(127.6.8.253:41923)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:11.677 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.8.253:41923: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:11.677 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923; will retry
16:20:11.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:11.993427 6499 master_runner.cc:186] Error getting master registration for 127.6.8.254:34321: Network error: Client connection negotiation failed: client connection to 127.6.8.254:34321: connect: Connection refused (error 111),
16:20:11.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:11.993772 6499 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:20:12.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:12.012451 6601 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:20:12.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:12.012781 6601 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:20:13.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:13.015606 6601 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:20:13.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:13.015919 6601 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:20:14.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:14.018740 6601 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:20:14.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:14.019037 6601 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:20:14.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:14.124406 6815 raft_consensus.cc:482] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
16:20:14.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:14.125051 6815 raft_consensus.cc:504] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9636eb75872e49cb90a026bad879c251" member_type: VOTER last_known_addr { host: "127.6.8.254" port: 34321 } } peers { permanent_uuid: "a297c0bf389d4a5ca444bd373be38e48" member_type: VOTER last_known_addr { host: "127.6.8.253" port: 41923 } } peers { permanent_uuid: "cb3c6ef4d12e40779471298ca16ad1fa" member_type: VOTER last_known_addr { host: "127.6.8.252" port: 39609 } }
16:20:14.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:14.127137 6815 leader_election.cc:288] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 9636eb75872e49cb90a026bad879c251 (127.6.8.254:34321), a297c0bf389d4a5ca444bd373be38e48 (127.6.8.253:41923)
16:20:14.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:14.129313 6523 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.8.254:34321: connect: Connection refused (error 111) [suppressed 3 similar messages]
16:20:14.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:14.132084 6522 leader_election.cc:334] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a297c0bf389d4a5ca444bd373be38e48 (127.6.8.253:41923): Network error: Client connection negotiation failed: client connection to 127.6.8.253:41923: connect: Connection refused (error 111)
16:20:14.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:14.132560 6523 leader_election.cc:334] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 9636eb75872e49cb90a026bad879c251 (127.6.8.254:34321): Network error: Client connection negotiation failed: client connection to 127.6.8.254:34321: connect: Connection refused (error 111)
16:20:14.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:14.132995 6523 leader_election.cc:302] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: cb3c6ef4d12e40779471298ca16ad1fa; no voters: 9636eb75872e49cb90a026bad879c251, a297c0bf389d4a5ca444bd373be38e48
16:20:14.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:14.133677 6815 raft_consensus.cc:2726] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
16:20:14.792 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.8.254:34321(127.6.8.254:34321)] connecting to peer
16:20:14.793 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.8.253:41923(127.6.8.253:41923)] connecting to peer
16:20:14.794 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:14.794 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:14.795 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.8.253:41923(127.6.8.253:41923)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:14.795 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.8.253:41923: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:14.795 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.8.254:34321(127.6.8.254:34321)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:14.795 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.8.254:34321: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:14.796 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923; will retry
16:20:15.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:15.021854 6601 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:20:15.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:15.022197 6601 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:20:16.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:16.026271 6601 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:20:16.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:16.026595 6601 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:20:16.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 16:20:16 dist-test-slave-cbbz krb5kdc[6192](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786465216, etypes {rep=17 tkt=17 ses=17}, kudu/127.6.8.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:20:16.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:16.674120 6515 init.cc:295] Successfully reacquired a new kerberos TGT
16:20:16.751 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.8.254:34321(127.6.8.254:34321)] connecting to peer
16:20:16.753 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.8.253:41923(127.6.8.253:41923)] connecting to peer
16:20:16.753 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:16.754 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.8.254:34321(127.6.8.254:34321)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:16.754 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.8.254:34321: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:16.757 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:16.757 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.8.253:41923(127.6.8.253:41923)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:16.757 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.8.253:41923: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:16.757 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923; will retry
16:20:16.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:16.997311 6499 master_runner.cc:186] Error getting master registration for 127.6.8.254:34321: Network error: Client connection negotiation failed: client connection to 127.6.8.254:34321: connect: Connection refused (error 111),
16:20:16.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:16.997829 6499 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:20:17.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:17.029454 6601 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:20:17.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:17.029758 6601 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:20:17.312 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.8.254:34321(127.6.8.254:34321)] connecting to peer
16:20:17.313 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.8.253:41923(127.6.8.253:41923)] connecting to peer
16:20:17.314 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:17.314 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.8.254:34321(127.6.8.254:34321)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:17.314 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:17.315 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.8.254:34321: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:17.315 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.8.253:41923(127.6.8.253:41923)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:17.315 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.8.253:41923: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:17.316 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923; will retry
16:20:18.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:18.032879 6601 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:20:18.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:18.033169 6601 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:20:18.331 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.8.254:34321(127.6.8.254:34321)] connecting to peer
16:20:18.332 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.8.253:41923(127.6.8.253:41923)] connecting to peer
16:20:18.333 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:18.334 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.8.254:34321(127.6.8.254:34321)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:18.336 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.8.254:34321: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:18.336 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:18.336 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.8.253:41923(127.6.8.253:41923)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:18.336 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.8.253:41923: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:18.336 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923; will retry
16:20:18.371 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.8.254:34321(127.6.8.254:34321)] connecting to peer
16:20:18.373 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.8.253:41923(127.6.8.253:41923)] connecting to peer
16:20:18.374 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:18.374 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.8.254:34321(127.6.8.254:34321)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:18.374 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.8.254:34321: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:18.375 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:18.375 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.8.253:41923(127.6.8.253:41923)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:18.376 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.8.253:41923: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:18.376 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923; will retry
16:20:19.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:19.035918 6601 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:20:19.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:19.036439 6601 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:20:20.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:20.039306 6601 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:20:20.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:20.039654 6601 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:20:21.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:21.042544 6601 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:20:21.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:21.042810 6601 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:20:21.872 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.8.254:34321(127.6.8.254:34321)] connecting to peer
16:20:21.872 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.8.253:41923(127.6.8.253:41923)] connecting to peer
16:20:21.873 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:21.874 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.8.254:34321(127.6.8.254:34321)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:21.874 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.8.254:34321: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:21.875 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:21.875 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.8.253:41923(127.6.8.253:41923)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:21.875 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.8.253:41923: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:21.876 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923; will retry
16:20:22.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:22.000941 6499 master_runner.cc:186] Error getting master registration for 127.6.8.254:34321: Network error: Client connection negotiation failed: client connection to 127.6.8.254:34321: connect: Connection refused (error 111),
16:20:22.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:22.001386 6499 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:20:22.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:22.045733 6601 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:20:22.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:22.046044 6601 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:20:22.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:22.776996 6835 raft_consensus.cc:482] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
16:20:22.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:22.777534 6835 raft_consensus.cc:504] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9636eb75872e49cb90a026bad879c251" member_type: VOTER last_known_addr { host: "127.6.8.254" port: 34321 } } peers { permanent_uuid: "a297c0bf389d4a5ca444bd373be38e48" member_type: VOTER last_known_addr { host: "127.6.8.253" port: 41923 } } peers { permanent_uuid: "cb3c6ef4d12e40779471298ca16ad1fa" member_type: VOTER last_known_addr { host: "127.6.8.252" port: 39609 } }
16:20:22.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:22.779331 6835 leader_election.cc:288] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 9636eb75872e49cb90a026bad879c251 (127.6.8.254:34321), a297c0bf389d4a5ca444bd373be38e48 (127.6.8.253:41923)
16:20:22.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:22.781883 6523 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.8.254:34321: connect: Connection refused (error 111) [suppressed 1 similar messages]
16:20:22.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:22.785181 6522 leader_election.cc:334] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a297c0bf389d4a5ca444bd373be38e48 (127.6.8.253:41923): Network error: Client connection negotiation failed: client connection to 127.6.8.253:41923: connect: Connection refused (error 111)
16:20:22.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:22.785636 6523 leader_election.cc:334] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 9636eb75872e49cb90a026bad879c251 (127.6.8.254:34321): Network error: Client connection negotiation failed: client connection to 127.6.8.254:34321: connect: Connection refused (error 111)
16:20:22.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:22.786028 6523 leader_election.cc:302] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: cb3c6ef4d12e40779471298ca16ad1fa; no voters: 9636eb75872e49cb90a026bad879c251, a297c0bf389d4a5ca444bd373be38e48
16:20:22.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:22.786972 6835 raft_consensus.cc:2726] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
16:20:23.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:23.049391 6601 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:20:23.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:23.049687 6601 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:20:24.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:24.052544 6601 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:20:24.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:24.052855 6601 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:20:25.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:25.055593 6601 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:20:25.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:25.055904 6601 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:20:25.511 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.8.254:34321(127.6.8.254:34321)] connecting to peer
16:20:25.512 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.8.253:41923(127.6.8.253:41923)] connecting to peer
16:20:25.513 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:25.513 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.8.253:41923(127.6.8.253:41923)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:25.513 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.8.253:41923: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:25.514 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:25.514 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.8.254:34321(127.6.8.254:34321)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:25.514 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.8.254:34321: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:25.516 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923; will retry
16:20:26.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:26.059196 6601 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:20:26.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:26.059597 6601 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:20:26.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 16:20:26 dist-test-slave-cbbz krb5kdc[6192](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786465226, etypes {rep=17 tkt=17 ses=17}, kudu/127.6.8.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:20:26.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:26.676853 6515 init.cc:295] Successfully reacquired a new kerberos TGT
16:20:27.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:27.004565 6499 master_runner.cc:186] Error getting master registration for 127.6.8.254:34321: Network error: Client connection negotiation failed: client connection to 127.6.8.254:34321: connect: Connection refused (error 111),
16:20:27.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:27.004954 6499 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:20:27.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:27.062338 6601 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:20:27.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:27.062639 6601 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:20:27.932 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.8.254:34321(127.6.8.254:34321)] connecting to peer
16:20:27.932 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.8.253:41923(127.6.8.253:41923)] connecting to peer
16:20:27.933 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:27.933 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.8.254:34321(127.6.8.254:34321)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:27.933 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.8.254:34321: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:27.933 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:27.934 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.8.253:41923(127.6.8.253:41923)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:27.934 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.8.253:41923: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:27.935 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923; will retry
16:20:28.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:28.065233 6601 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:20:28.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:28.065518 6601 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:20:29.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:29.067970 6601 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cb3c6ef4d12e40779471298ca16ad1fa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:20:29.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:29.068239 6601 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:20:29.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:29.415059 6578 debug-util.cc:398] Leaking SignalData structure 0x7b0800088aa0 after lost signal to thread 6500
16:20:29.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 16:20:29.415829 6578 debug-util.cc:398] Leaking SignalData structure 0x7b080009a5e0 after lost signal to thread 6582
16:20:29.552 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.8.254:34321(127.6.8.254:34321)] connecting to peer
16:20:29.552 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.8.253:41923(127.6.8.253:41923)] connecting to peer
16:20:29.553 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:29.553 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:29.553 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.8.254:34321(127.6.8.254:34321)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:29.554 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.8.254:34321: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321
16:20:29.553 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.8.253:41923(127.6.8.253:41923)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:29.555 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.8.253:41923: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
16:20:29.557 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923; will retry
16:20:29.561 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=28827), Traces: [0ms] refreshing cache from master, [35ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.252:39609, [107ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.254:34321, [117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.253:41923, [158ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.254:34321: Network error: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321, [232ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.253:41923: Network error: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [504ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.252:39609: OK, [509ms] delaying RPC due to: Service unavailable: Master config (127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [528ms] refreshing cache from master, [529ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.252:39609, [531ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.254:34321, [534ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.253:41923, [535ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.254:34321: Network error: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321, [536ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.252:39609: OK, [538ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.253:41923: Network error: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [539ms] delaying RPC due to: Service unavailable: Master config (127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [547ms] refreshing cache from master, [547ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.252:39609, [548ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.254:34321, [549ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.253:41923, [552ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.254:34321: Network error: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321, [553ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.252:39609: OK, [553ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.253:41923: Network error: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [554ms] delaying RPC due to: Service unavailable: Master config (127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [568ms] refreshing cache from master, [568ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.252:39609, [569ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.254:34321, [570ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.253:41923, [573ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.254:34321: Network error: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321, [574ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.252:39609: OK, [575ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.253:41923: Network error: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [576ms] delaying RPC due to: Service unavailable: Master config (127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [587ms] refreshing cache from master, [588ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.252:39609, [588ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.254:34321, [589ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.253:41923, [592ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.254:34321: Network error: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321, [593ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.252:39609: OK, [594ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.253:41923: Network error: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [595ms] delaying RPC due to: Service unavailable: Master config (127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [628ms] refreshing cache from master, [628ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.252:39609, [629ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.254:34321, [630ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.253:41923, [632ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.254:34321: Network error: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321, [636ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.252:39609: OK, [637ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.253:41923: Network error: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [639ms] delaying RPC due to: Service unavailable: Master config (127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [707ms] refreshing cache from master, [708ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.252:39609, [708ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.254:34321, [710ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.253:41923, [713ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.254:34321: Network error: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321, [714ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.252:39609: OK, [715ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.253:41923: Network error: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [716ms] delaying RPC due to: Service unavailable: Master config (127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [808ms] refreshing cache from master, [808ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.252:39609, [808ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.254:34321, [810ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.253:41923, [812ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.254:34321: Network error: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321, [812ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.252:39609: OK, [815ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.253:41923: Network error: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [816ms] delaying RPC due to: Service unavailable: Master config (127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [887ms] refreshing cache from master, [888ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.252:39609, [888ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.254:34321, [889ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.253:41923, [891ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.254:34321: Network error: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321, [893ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.252:39609: OK, [894ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.253:41923: Network error: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [896ms] delaying RPC due to: Service unavailable: Master config (127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [1328ms] refreshing cache from master, [1328ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.252:39609, [1329ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.254:34321, [1330ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.253:41923, [1332ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.254:34321: Network error: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321, [1334ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.252:39609: OK, [1335ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.253:41923: Network error: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [1336ms] delaying RPC due to: Service unavailable: Master config (127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [1588ms] refreshing cache from master, [1588ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.252:39609, [1588ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.254:34321, [1590ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.253:41923, [1592ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.254:34321: Network error: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321, [1592ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.252:39609: OK, [1594ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.253:41923: Network error: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [1594ms] delaying RPC due to: Service unavailable: Master config (127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [2248ms] refreshing cache from master, [2248ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.252:39609, [2249ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.254:34321, [2251ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.253:41923, [2251ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.254:34321: Network error: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321, [2252ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.252:39609: OK, [2254ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.253:41923: Network error: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [2254ms] delaying RPC due to: Service unavailable: Master config (127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [5908ms] refreshing cache from master, [5908ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.252:39609, [5908ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.254:34321, [5910ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.253:41923, [5913ms] too many traces: truncated at 100 traces, deferred=Deferred@1464865011(state=PENDING, result=null, callback=(continuation of Deferred@563122931 after retry RPC after error@294097999) -> (continuation of Deferred@828798117 after retry RPC after error@1726365170) -> (continuation of Deferred@1128541399 after retry RPC after error@1850535196) -> (continuation of Deferred@1812823852 after retry RPC after error@1003734770) -> (continuation of Deferred@1502836683 after retry RPC after error@1354651112) -> (continuation of Deferred@1473098956 after retry RPC after error@1921784977) -> (continuation of Deferred@811798564 after retry RPC after error@315622101) -> (continuation of Deferred@671203223 after retry RPC after error@981161618) -> (continuation of Deferred@305238985 after retry RPC after error@6684144) -> (continuation of Deferred@624760684 after retry RPC after error@1167202975) -> (continuation of Deferred@1537435199 after retry RPC after error@1954974406) -> (continuation of Deferred@969766221 after retry RPC after error@1818150409) -> (continuation of Deferred@739034915 after retry RPC after error@1822210270) -> (continuation of Deferred@1042931485 after retry RPC after error@618477663) -> (continuation of Deferred@1847772728 after retry RPC after error@1735514474) -> (continuation of Deferred@399520686 after retry RPC after error@1518530251) -> (continuation of Deferred@585756837 after retry RPC after error@1270531867) -> (continuation of Deferred@1566691457 after retry RPC after error@282630480) -> (continuation of Deferred@2111385398 after retry RPC after error@1615248777) -> (continuation of Deferred@871402712 after retry RPC after error@1731625455) -> (continuation of Deferred@1394778784 after retry RPC after error@115210168) -> (continuation of Deferred@838610423 after retry RPC after error@1654062046) -> (continuation of Deferred@619147918 after retry RPC after error@1299067293), errback=(continuation of Deferred@563122931 after retry RPC after error@294097999) -> (continuation of Deferred@828798117 after retry RPC after error@1726365170) -> (continuation of Deferred@1128541399 after retry RPC after error@1850535196) -> (continuation of Deferred@1812823852 after retry RPC after error@1003734770) -> (continuation of Deferred@1502836683 after retry RPC after error@1354651112) -> (continuation of Deferred@1473098956 after retry RPC after error@1921784977) -> (continuation of Deferred@811798564 after retry RPC after error@315622101) -> (continuation of Deferred@671203223 after retry RPC after error@981161618) -> (continuation of Deferred@305238985 after retry RPC after error@6684144) -> (continuation of Deferred@624760684 after retry RPC after error@1167202975) -> (continuation of Deferred@1537435199 after retry RPC after error@1954974406) -> (continuation of Deferred@969766221 after retry RPC after error@1818150409) -> (continuation of Deferred@739034915 after retry RPC after error@1822210270) -> (continuation of Deferred@1042931485 after retry RPC after error@618477663) -> (continuation of Deferred@1847772728 after retry RPC after error@1735514474) -> (continuation of Deferred@399520686 after retry RPC after error@1518530251) -> (continuation of Deferred@585756837 after retry RPC after error@1270531867) -> (continuation of Deferred@1566691457 after retry RPC after error@282630480) -> (continuation of Deferred@2111385398 after retry RPC after error@1615248777) -> (continuation of Deferred@871402712 after retry RPC after error@1731625455) -> (continuation of Deferred@1394778784 after retry RPC after error@115210168) -> (continuation of Deferred@838610423 after retry RPC after error@1654062046) -> (continuation of Deferred@619147918 after retry RPC after error@1299067293)))
16:20:29.567 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.6.8.252:39609(127.6.8.252:39609)] disconnecting while in state READY
16:20:29.568 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.6.8.253:41923(127.6.8.253:41923)] disconnecting while in state TERMINATED
16:20:29.568 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.6.8.254:34321(127.6.8.254:34321)] disconnecting while in state TERMINATED
16:20:29.574 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
16:20:29.576 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
16:20:29.577 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.6.8.252:39609(127.6.8.252:39609)] handling channelInactive
16:20:29.577 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.8.252:39609(127.6.8.252:39609)] cleaning up while in state READY due to: connection closed
16:20:29.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:29.586131 6179 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVGVFlJ/build/tsan/bin/kudu with pid 6499
16:20:30.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-11T16:20:30Z chronyd exiting
16:20:31.825 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
16:20:31.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:31.826426 5368 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVGVFlJ/build/tsan/bin/kudu with pid 5681
16:20:31.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:31.857888 5368 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVGVFlJ/build/tsan/bin/kudu with pid 5865
16:20:31.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:31.900110 5368 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVGVFlJ/build/tsan/bin/kudu with pid 6014
16:20:31.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:31.941036 5368 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVGVFlJ/build/tsan/bin/kudu with pid 5382
16:20:31.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:31.978438 5368 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVGVFlJ/build/tsan/bin/kudu with pid 5460
16:20:32.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 16:20:32.014837 5368 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVGVFlJ/build/tsan/bin/kudu with pid 5544
16:20:33.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-11T16:20:33Z chronyd exiting
16:20:34.214 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=28827), Traces: [0ms] refreshing cache from master, [35ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.252:39609, [107ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.254:34321, [117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.253:41923, [158ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.254:34321: Network error: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321, [232ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.253:41923: Network error: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [504ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.252:39609: OK, [509ms] delaying RPC due to: Service unavailable: Master config (127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [528ms] refreshing cache from master, [529ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.252:39609, [531ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.254:34321, [534ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.253:41923, [535ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.254:34321: Network error: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321, [536ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.252:39609: OK, [538ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.253:41923: Network error: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [539ms] delaying RPC due to: Service unavailable: Master config (127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [547ms] refreshing cache from master, [547ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.252:39609, [548ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.254:34321, [549ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.253:41923, [552ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.254:34321: Network error: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321, [553ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.252:39609: OK, [553ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.253:41923: Network error: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [554ms] delaying RPC due to: Service unavailable: Master config (127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [568ms] refreshing cache from master, [568ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.252:39609, [569ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.254:34321, [570ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.253:41923, [573ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.254:34321: Network error: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321, [574ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.252:39609: OK, [575ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.253:41923: Network error: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [576ms] delaying RPC due to: Service unavailable: Master config (127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [587ms] refreshing cache from master, [588ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.252:39609, [588ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.254:34321, [589ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.253:41923, [592ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.254:34321: Network error: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321, [593ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.252:39609: OK, [594ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.253:41923: Network error: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [595ms] delaying RPC due to: Service unavailable: Master config (127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [628ms] refreshing cache from master, [628ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.252:39609, [629ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.254:34321, [630ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.253:41923, [632ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.254:34321: Network error: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321, [636ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.252:39609: OK, [637ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.253:41923: Network error: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [639ms] delaying RPC due to: Service unavailable: Master config (127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [707ms] refreshing cache from master, [708ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.252:39609, [708ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.254:34321, [710ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.253:41923, [713ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.254:34321: Network error: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321, [714ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.252:39609: OK, [715ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.253:41923: Network error: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [716ms] delaying RPC due to: Service unavailable: Master config (127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [808ms] refreshing cache from master, [808ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.252:39609, [808ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.254:34321, [810ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.253:41923, [812ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.254:34321: Network error: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321, [812ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.252:39609: OK, [815ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.253:41923: Network error: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [816ms] delaying RPC due to: Service unavailable: Master config (127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [887ms] refreshing cache from master, [888ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.252:39609, [888ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.254:34321, [889ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.253:41923, [891ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.254:34321: Network error: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321, [893ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.252:39609: OK, [894ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.253:41923: Network error: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [896ms] delaying RPC due to: Service unavailable: Master config (127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [1328ms] refreshing cache from master, [1328ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.252:39609, [1329ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.254:34321, [1330ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.253:41923, [1332ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.254:34321: Network error: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321, [1334ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.252:39609: OK, [1335ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.253:41923: Network error: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [1336ms] delaying RPC due to: Service unavailable: Master config (127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [1588ms] refreshing cache from master, [1588ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.252:39609, [1588ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.254:34321, [1590ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.253:41923, [1592ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.254:34321: Network error: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321, [1592ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.252:39609: OK, [1594ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.253:41923: Network error: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [1594ms] delaying RPC due to: Service unavailable: Master config (127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [2248ms] refreshing cache from master, [2248ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.252:39609, [2249ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.254:34321, [2251ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.253:41923, [2251ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.254:34321: Network error: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321, [2252ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.252:39609: OK, [2254ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.253:41923: Network error: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [2254ms] delaying RPC due to: Service unavailable: Master config (127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [5908ms] refreshing cache from master, [5908ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.252:39609, [5908ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.254:34321, [5910ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.253:41923, [5913ms] too many traces: truncated at 100 traces, deferred=Deferred@1464865011(state=PENDING, result=null, callback=(continuation of Deferred@563122931 after retry RPC after error@294097999) -> (continuation of Deferred@828798117 after retry RPC after error@1726365170) -> (continuation of Deferred@1128541399 after retry RPC after error@1850535196) -> (continuation of Deferred@1812823852 after retry RPC after error@1003734770) -> (continuation of Deferred@1502836683 after retry RPC after error@1354651112) -> (continuation of Deferred@1473098956 after retry RPC after error@1921784977) -> (continuation of Deferred@811798564 after retry RPC after error@315622101) -> (continuation of Deferred@671203223 after retry RPC after error@981161618) -> (continuation of Deferred@305238985 after retry RPC after error@6684144) -> (continuation of Deferred@624760684 after retry RPC after error@1167202975) -> (continuation of Deferred@1537435199 after retry RPC after error@1954974406) -> (continuation of Deferred@969766221 after retry RPC after error@1818150409) -> (continuation of Deferred@739034915 after retry RPC after error@1822210270) -> (continuation of Deferred@1042931485 after retry RPC after error@618477663) -> (continuation of Deferred@1847772728 after retry RPC after error@1735514474) -> (continuation of Deferred@399520686 after retry RPC after error@1518530251) -> (continuation of Deferred@585756837 after retry RPC after error@1270531867) -> (continuation of Deferred@1566691457 after retry RPC after error@282630480) -> (continuation of Deferred@2111385398 after retry RPC after error@1615248777) -> (continuation of Deferred@871402712 after retry RPC after error@1731625455) -> (continuation of Deferred@1394778784 after retry RPC after error@115210168) -> (continuation of Deferred@838610423 after retry RPC after error@1654062046) -> (continuation of Deferred@619147918 after retry RPC after error@1299067293), errback=(continuation of Deferred@563122931 after retry RPC after error@294097999) -> (continuation of Deferred@828798117 after retry RPC after error@1726365170) -> (continuation of Deferred@1128541399 after retry RPC after error@1850535196) -> (continuation of Deferred@1812823852 after retry RPC after error@1003734770) -> (continuation of Deferred@1502836683 after retry RPC after error@1354651112) -> (continuation of Deferred@1473098956 after retry RPC after error@1921784977) -> (continuation of Deferred@811798564 after retry RPC after error@315622101) -> (continuation of Deferred@671203223 after retry RPC after error@981161618) -> (continuation of Deferred@305238985 after retry RPC after error@6684144) -> (continuation of Deferred@624760684 after retry RPC after error@1167202975) -> (continuation of Deferred@1537435199 after retry RPC after error@1954974406) -> (continuation of Deferred@969766221 after retry RPC after error@1818150409) -> (continuation of Deferred@739034915 after retry RPC after error@1822210270) -> (continuation of Deferred@1042931485 after retry RPC after error@618477663) -> (continuation of Deferred@1847772728 after retry RPC after error@1735514474) -> (continuation of Deferred@399520686 after retry RPC after error@1518530251) -> (continuation of Deferred@585756837 after retry RPC after error@1270531867) -> (continuation of Deferred@1566691457 after retry RPC after error@282630480) -> (continuation of Deferred@2111385398 after retry RPC after error@1615248777) -> (continuation of Deferred@871402712 after retry RPC after error@1731625455) -> (continuation of Deferred@1394778784 after retry RPC after error@115210168) -> (continuation of Deferred@838610423 after retry RPC after error@1654062046) -> (continuation of Deferred@619147918 after retry RPC after error@1299067293)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.6.8.253:41923
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=28827), Traces: [0ms] refreshing cache from master, [35ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.252:39609, [107ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.254:34321, [117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.253:41923, [158ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.254:34321: Network error: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321, [232ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.253:41923: Network error: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [504ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.252:39609: OK, [509ms] delaying RPC due to: Service unavailable: Master config (127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [528ms] refreshing cache from master, [529ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.252:39609, [531ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.254:34321, [534ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.253:41923, [535ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.254:34321: Network error: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321, [536ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.252:39609: OK, [538ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.253:41923: Network error: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [539ms] delaying RPC due to: Service unavailable: Master config (127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [547ms] refreshing cache from master, [547ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.252:39609, [548ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.254:34321, [549ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.253:41923, [552ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.254:34321: Network error: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321, [553ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.252:39609: OK, [553ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.253:41923: Network error: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [554ms] delaying RPC due to: Service unavailable: Master config (127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [568ms] refreshing cache from master, [568ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.252:39609, [569ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.254:34321, [570ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.253:41923, [573ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.254:34321: Network error: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321, [574ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.252:39609: OK, [575ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.253:41923: Network error: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [576ms] delaying RPC due to: Service unavailable: Master config (127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [587ms] refreshing cache from master, [588ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.252:39609, [588ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.254:34321, [589ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.253:41923, [592ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.254:34321: Network error: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321, [593ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.252:39609: OK, [594ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.253:41923: Network error: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [595ms] delaying RPC due to: Service unavailable: Master config (127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [628ms] refreshing cache from master, [628ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.252:39609, [629ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.254:34321, [630ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.253:41923, [632ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.254:34321: Network error: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321, [636ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.252:39609: OK, [637ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.253:41923: Network error: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [639ms] delaying RPC due to: Service unavailable: Master config (127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [707ms] refreshing cache from master, [708ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.252:39609, [708ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.254:34321, [710ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.253:41923, [713ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.254:34321: Network error: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321, [714ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.252:39609: OK, [715ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.253:41923: Network error: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [716ms] delaying RPC due to: Service unavailable: Master config (127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [808ms] refreshing cache from master, [808ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.252:39609, [808ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.254:34321, [810ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.253:41923, [812ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.254:34321: Network error: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321, [812ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.252:39609: OK, [815ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.253:41923: Network error: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [816ms] delaying RPC due to: Service unavailable: Master config (127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [887ms] refreshing cache from master, [888ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.252:39609, [888ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.254:34321, [889ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.253:41923, [891ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.254:34321: Network error: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321, [893ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.252:39609: OK, [894ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.253:41923: Network error: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [896ms] delaying RPC due to: Service unavailable: Master config (127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [1328ms] refreshing cache from master, [1328ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.252:39609, [1329ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.254:34321, [1330ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.253:41923, [1332ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.254:34321: Network error: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321, [1334ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.252:39609: OK, [1335ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.253:41923: Network error: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [1336ms] delaying RPC due to: Service unavailable: Master config (127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [1588ms] refreshing cache from master, [1588ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.252:39609, [1588ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.254:34321, [1590ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.253:41923, [1592ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.254:34321: Network error: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321, [1592ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.252:39609: OK, [1594ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.253:41923: Network error: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [1594ms] delaying RPC due to: Service unavailable: Master config (127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [2248ms] refreshing cache from master, [2248ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.252:39609, [2249ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.254:34321, [2251ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.253:41923, [2251ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.254:34321: Network error: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321, [2252ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.252:39609: OK, [2254ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.253:41923: Network error: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [2254ms] delaying RPC due to: Service unavailable: Master config (127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [5908ms] refreshing cache from master, [5908ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.252:39609, [5908ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.254:34321, [5910ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.253:41923, [5913ms] too many traces: truncated at 100 traces, deferred=Deferred@1464865011(state=PENDING, result=null, callback=(continuation of Deferred@563122931 after retry RPC after error@294097999) -> (continuation of Deferred@828798117 after retry RPC after error@1726365170) -> (continuation of Deferred@1128541399 after retry RPC after error@1850535196) -> (continuation of Deferred@1812823852 after retry RPC after error@1003734770) -> (continuation of Deferred@1502836683 after retry RPC after error@1354651112) -> (continuation of Deferred@1473098956 after retry RPC after error@1921784977) -> (continuation of Deferred@811798564 after retry RPC after error@315622101) -> (continuation of Deferred@671203223 after retry RPC after error@981161618) -> (continuation of Deferred@305238985 after retry RPC after error@6684144) -> (continuation of Deferred@624760684 after retry RPC after error@1167202975) -> (continuation of Deferred@1537435199 after retry RPC after error@1954974406) -> (continuation of Deferred@969766221 after retry RPC after error@1818150409) -> (continuation of Deferred@739034915 after retry RPC after error@1822210270) -> (continuation of Deferred@1042931485 after retry RPC after error@618477663) -> (continuation of Deferred@1847772728 after retry RPC after error@1735514474) -> (continuation of Deferred@399520686 after retry RPC after error@1518530251) -> (continuation of Deferred@585756837 after retry RPC after error@1270531867) -> (continuation of Deferred@1566691457 after retry RPC after error@282630480) -> (continuation of Deferred@2111385398 after retry RPC after error@1615248777) -> (continuation of Deferred@871402712 after retry RPC after error@1731625455) -> (continuation of Deferred@1394778784 after retry RPC after error@115210168) -> (continuation of Deferred@838610423 after retry RPC after error@1654062046) -> (continuation of Deferred@619147918 after retry RPC after error@1299067293), errback=(continuation of Deferred@563122931 after retry RPC after error@294097999) -> (continuation of Deferred@828798117 after retry RPC after error@1726365170) -> (continuation of Deferred@1128541399 after retry RPC after error@1850535196) -> (continuation of Deferred@1812823852 after retry RPC after error@1003734770) -> (continuation of Deferred@1502836683 after retry RPC after error@1354651112) -> (continuation of Deferred@1473098956 after retry RPC after error@1921784977) -> (continuation of Deferred@811798564 after retry RPC after error@315622101) -> (continuation of Deferred@671203223 after retry RPC after error@981161618) -> (continuation of Deferred@305238985 after retry RPC after error@6684144) -> (continuation of Deferred@624760684 after retry RPC after error@1167202975) -> (continuation of Deferred@1537435199 after retry RPC after error@1954974406) -> (continuation of Deferred@969766221 after retry RPC after error@1818150409) -> (continuation of Deferred@739034915 after retry RPC after error@1822210270) -> (continuation of Deferred@1042931485 after retry RPC after error@618477663) -> (continuation of Deferred@1847772728 after retry RPC after error@1735514474) -> (continuation of Deferred@399520686 after retry RPC after error@1518530251) -> (continuation of Deferred@585756837 after retry RPC after error@1270531867) -> (continuation of Deferred@1566691457 after retry RPC after error@282630480) -> (continuation of Deferred@2111385398 after retry RPC after error@1615248777) -> (continuation of Deferred@871402712 after retry RPC after error@1731625455) -> (continuation of Deferred@1394778784 after retry RPC after error@115210168) -> (continuation of Deferred@838610423 after retry RPC after error@1654062046) -> (continuation of Deferred@619147918 after retry RPC after error@1299067293)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.6.8.253:41923
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
16:20:35.038 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
16:20:35.038 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=28827), Traces: [0ms] refreshing cache from master, [35ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.252:39609, [107ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.254:34321, [117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.253:41923, [158ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.254:34321: Network error: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321, [232ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.253:41923: Network error: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [504ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.252:39609: OK, [509ms] delaying RPC due to: Service unavailable: Master config (127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [528ms] refreshing cache from master, [529ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.252:39609, [531ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.254:34321, [534ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.253:41923, [535ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.254:34321: Network error: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321, [536ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.252:39609: OK, [538ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.253:41923: Network error: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [539ms] delaying RPC due to: Service unavailable: Master config (127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [547ms] refreshing cache from master, [547ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.252:39609, [548ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.254:34321, [549ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.253:41923, [552ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.254:34321: Network error: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321, [553ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.252:39609: OK, [553ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.253:41923: Network error: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [554ms] delaying RPC due to: Service unavailable: Master config (127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [568ms] refreshing cache from master, [568ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.252:39609, [569ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.254:34321, [570ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.253:41923, [573ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.254:34321: Network error: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321, [574ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.252:39609: OK, [575ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.253:41923: Network error: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [576ms] delaying RPC due to: Service unavailable: Master config (127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [587ms] refreshing cache from master, [588ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.252:39609, [588ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.254:34321, [589ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.253:41923, [592ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.254:34321: Network error: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321, [593ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.252:39609: OK, [594ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.253:41923: Network error: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [595ms] delaying RPC due to: Service unavailable: Master config (127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [628ms] refreshing cache from master, [628ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.252:39609, [629ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.254:34321, [630ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.253:41923, [632ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.254:34321: Network error: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321, [636ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.252:39609: OK, [637ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.253:41923: Network error: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [639ms] delaying RPC due to: Service unavailable: Master config (127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [707ms] refreshing cache from master, [708ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.252:39609, [708ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.254:34321, [710ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.253:41923, [713ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.254:34321: Network error: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321, [714ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.252:39609: OK, [715ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.253:41923: Network error: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [716ms] delaying RPC due to: Service unavailable: Master config (127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [808ms] refreshing cache from master, [808ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.252:39609, [808ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.254:34321, [810ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.253:41923, [812ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.254:34321: Network error: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321, [812ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.252:39609: OK, [815ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.253:41923: Network error: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [816ms] delaying RPC due to: Service unavailable: Master config (127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [887ms] refreshing cache from master, [888ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.252:39609, [888ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.254:34321, [889ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.253:41923, [891ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.254:34321: Network error: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321, [893ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.252:39609: OK, [894ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.253:41923: Network error: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [896ms] delaying RPC due to: Service unavailable: Master config (127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [1328ms] refreshing cache from master, [1328ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.252:39609, [1329ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.254:34321, [1330ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.253:41923, [1332ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.254:34321: Network error: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321, [1334ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.252:39609: OK, [1335ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.253:41923: Network error: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [1336ms] delaying RPC due to: Service unavailable: Master config (127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [1588ms] refreshing cache from master, [1588ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.252:39609, [1588ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.254:34321, [1590ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.253:41923, [1592ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.254:34321: Network error: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321, [1592ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.252:39609: OK, [1594ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.253:41923: Network error: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [1594ms] delaying RPC due to: Service unavailable: Master config (127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [2248ms] refreshing cache from master, [2248ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.252:39609, [2249ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.254:34321, [2251ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.253:41923, [2251ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.254:34321: Network error: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321, [2252ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.252:39609: OK, [2254ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.253:41923: Network error: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [2254ms] delaying RPC due to: Service unavailable: Master config (127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [5908ms] refreshing cache from master, [5908ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.252:39609, [5908ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.254:34321, [5910ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.253:41923, [5913ms] too many traces: truncated at 100 traces, deferred=Deferred@1464865011(state=PENDING, result=null, callback=(continuation of Deferred@563122931 after retry RPC after error@294097999) -> (continuation of Deferred@828798117 after retry RPC after error@1726365170) -> (continuation of Deferred@1128541399 after retry RPC after error@1850535196) -> (continuation of Deferred@1812823852 after retry RPC after error@1003734770) -> (continuation of Deferred@1502836683 after retry RPC after error@1354651112) -> (continuation of Deferred@1473098956 after retry RPC after error@1921784977) -> (continuation of Deferred@811798564 after retry RPC after error@315622101) -> (continuation of Deferred@671203223 after retry RPC after error@981161618) -> (continuation of Deferred@305238985 after retry RPC after error@6684144) -> (continuation of Deferred@624760684 after retry RPC after error@1167202975) -> (continuation of Deferred@1537435199 after retry RPC after error@1954974406) -> (continuation of Deferred@969766221 after retry RPC after error@1818150409) -> (continuation of Deferred@739034915 after retry RPC after error@1822210270) -> (continuation of Deferred@1042931485 after retry RPC after error@618477663) -> (continuation of Deferred@1847772728 after retry RPC after error@1735514474) -> (continuation of Deferred@399520686 after retry RPC after error@1518530251) -> (continuation of Deferred@585756837 after retry RPC after error@1270531867) -> (continuation of Deferred@1566691457 after retry RPC after error@282630480) -> (continuation of Deferred@2111385398 after retry RPC after error@1615248777) -> (continuation of Deferred@871402712 after retry RPC after error@1731625455) -> (continuation of Deferred@1394778784 after retry RPC after error@115210168) -> (continuation of Deferred@838610423 after retry RPC after error@1654062046) -> (continuation of Deferred@619147918 after retry RPC after error@1299067293), errback=(continuation of Deferred@563122931 after retry RPC after error@294097999) -> (continuation of Deferred@828798117 after retry RPC after error@1726365170) -> (continuation of Deferred@1128541399 after retry RPC after error@1850535196) -> (continuation of Deferred@1812823852 after retry RPC after error@1003734770) -> (continuation of Deferred@1502836683 after retry RPC after error@1354651112) -> (continuation of Deferred@1473098956 after retry RPC after error@1921784977) -> (continuation of Deferred@811798564 after retry RPC after error@315622101) -> (continuation of Deferred@671203223 after retry RPC after error@981161618) -> (continuation of Deferred@305238985 after retry RPC after error@6684144) -> (continuation of Deferred@624760684 after retry RPC after error@1167202975) -> (continuation of Deferred@1537435199 after retry RPC after error@1954974406) -> (continuation of Deferred@969766221 after retry RPC after error@1818150409) -> (continuation of Deferred@739034915 after retry RPC after error@1822210270) -> (continuation of Deferred@1042931485 after retry RPC after error@618477663) -> (continuation of Deferred@1847772728 after retry RPC after error@1735514474) -> (continuation of Deferred@399520686 after retry RPC after error@1518530251) -> (continuation of Deferred@585756837 after retry RPC after error@1270531867) -> (continuation of Deferred@1566691457 after retry RPC after error@282630480) -> (continuation of Deferred@2111385398 after retry RPC after error@1615248777) -> (continuation of Deferred@871402712 after retry RPC after error@1731625455) -> (continuation of Deferred@1394778784 after retry RPC after error@115210168) -> (continuation of Deferred@838610423 after retry RPC after error@1654062046) -> (continuation of Deferred@619147918 after retry RPC after error@1299067293)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.6.8.253:41923
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=28827), Traces: [0ms] refreshing cache from master, [35ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.252:39609, [107ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.254:34321, [117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.253:41923, [158ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.254:34321: Network error: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321, [232ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.253:41923: Network error: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [504ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.252:39609: OK, [509ms] delaying RPC due to: Service unavailable: Master config (127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [528ms] refreshing cache from master, [529ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.252:39609, [531ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.254:34321, [534ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.253:41923, [535ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.254:34321: Network error: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321, [536ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.252:39609: OK, [538ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.253:41923: Network error: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [539ms] delaying RPC due to: Service unavailable: Master config (127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [547ms] refreshing cache from master, [547ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.252:39609, [548ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.254:34321, [549ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.253:41923, [552ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.254:34321: Network error: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321, [553ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.252:39609: OK, [553ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.253:41923: Network error: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [554ms] delaying RPC due to: Service unavailable: Master config (127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [568ms] refreshing cache from master, [568ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.252:39609, [569ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.254:34321, [570ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.253:41923, [573ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.254:34321: Network error: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321, [574ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.252:39609: OK, [575ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.253:41923: Network error: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [576ms] delaying RPC due to: Service unavailable: Master config (127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [587ms] refreshing cache from master, [588ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.252:39609, [588ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.254:34321, [589ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.253:41923, [592ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.254:34321: Network error: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321, [593ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.252:39609: OK, [594ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.253:41923: Network error: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [595ms] delaying RPC due to: Service unavailable: Master config (127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [628ms] refreshing cache from master, [628ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.252:39609, [629ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.254:34321, [630ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.253:41923, [632ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.254:34321: Network error: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321, [636ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.252:39609: OK, [637ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.253:41923: Network error: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [639ms] delaying RPC due to: Service unavailable: Master config (127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [707ms] refreshing cache from master, [708ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.252:39609, [708ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.254:34321, [710ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.253:41923, [713ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.254:34321: Network error: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321, [714ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.252:39609: OK, [715ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.253:41923: Network error: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [716ms] delaying RPC due to: Service unavailable: Master config (127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [808ms] refreshing cache from master, [808ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.252:39609, [808ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.254:34321, [810ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.253:41923, [812ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.254:34321: Network error: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321, [812ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.252:39609: OK, [815ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.253:41923: Network error: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [816ms] delaying RPC due to: Service unavailable: Master config (127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [887ms] refreshing cache from master, [888ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.252:39609, [888ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.254:34321, [889ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.253:41923, [891ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.254:34321: Network error: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321, [893ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.252:39609: OK, [894ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.253:41923: Network error: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [896ms] delaying RPC due to: Service unavailable: Master config (127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [1328ms] refreshing cache from master, [1328ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.252:39609, [1329ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.254:34321, [1330ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.253:41923, [1332ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.254:34321: Network error: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321, [1334ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.252:39609: OK, [1335ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.253:41923: Network error: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [1336ms] delaying RPC due to: Service unavailable: Master config (127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [1588ms] refreshing cache from master, [1588ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.252:39609, [1588ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.254:34321, [1590ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.253:41923, [1592ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.254:34321: Network error: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321, [1592ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.252:39609: OK, [1594ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.253:41923: Network error: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [1594ms] delaying RPC due to: Service unavailable: Master config (127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [2248ms] refreshing cache from master, [2248ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.252:39609, [2249ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.254:34321, [2251ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.253:41923, [2251ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.254:34321: Network error: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321, [2252ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.252:39609: OK, [2254ms] Sub RPC ConnectToMaster: received response from server master-127.6.8.253:41923: Network error: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [2254ms] delaying RPC due to: Service unavailable: Master config (127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923, [5908ms] refreshing cache from master, [5908ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.252:39609, [5908ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.254:34321, [5910ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.8.253:41923, [5913ms] too many traces: truncated at 100 traces, deferred=Deferred@1464865011(state=PENDING, result=null, callback=(continuation of Deferred@563122931 after retry RPC after error@294097999) -> (continuation of Deferred@828798117 after retry RPC after error@1726365170) -> (continuation of Deferred@1128541399 after retry RPC after error@1850535196) -> (continuation of Deferred@1812823852 after retry RPC after error@1003734770) -> (continuation of Deferred@1502836683 after retry RPC after error@1354651112) -> (continuation of Deferred@1473098956 after retry RPC after error@1921784977) -> (continuation of Deferred@811798564 after retry RPC after error@315622101) -> (continuation of Deferred@671203223 after retry RPC after error@981161618) -> (continuation of Deferred@305238985 after retry RPC after error@6684144) -> (continuation of Deferred@624760684 after retry RPC after error@1167202975) -> (continuation of Deferred@1537435199 after retry RPC after error@1954974406) -> (continuation of Deferred@969766221 after retry RPC after error@1818150409) -> (continuation of Deferred@739034915 after retry RPC after error@1822210270) -> (continuation of Deferred@1042931485 after retry RPC after error@618477663) -> (continuation of Deferred@1847772728 after retry RPC after error@1735514474) -> (continuation of Deferred@399520686 after retry RPC after error@1518530251) -> (continuation of Deferred@585756837 after retry RPC after error@1270531867) -> (continuation of Deferred@1566691457 after retry RPC after error@282630480) -> (continuation of Deferred@2111385398 after retry RPC after error@1615248777) -> (continuation of Deferred@871402712 after retry RPC after error@1731625455) -> (continuation of Deferred@1394778784 after retry RPC after error@115210168) -> (continuation of Deferred@838610423 after retry RPC after error@1654062046) -> (continuation of Deferred@619147918 after retry RPC after error@1299067293), errback=(continuation of Deferred@563122931 after retry RPC after error@294097999) -> (continuation of Deferred@828798117 after retry RPC after error@1726365170) -> (continuation of Deferred@1128541399 after retry RPC after error@1850535196) -> (continuation of Deferred@1812823852 after retry RPC after error@1003734770) -> (continuation of Deferred@1502836683 after retry RPC after error@1354651112) -> (continuation of Deferred@1473098956 after retry RPC after error@1921784977) -> (continuation of Deferred@811798564 after retry RPC after error@315622101) -> (continuation of Deferred@671203223 after retry RPC after error@981161618) -> (continuation of Deferred@305238985 after retry RPC after error@6684144) -> (continuation of Deferred@624760684 after retry RPC after error@1167202975) -> (continuation of Deferred@1537435199 after retry RPC after error@1954974406) -> (continuation of Deferred@969766221 after retry RPC after error@1818150409) -> (continuation of Deferred@739034915 after retry RPC after error@1822210270) -> (continuation of Deferred@1042931485 after retry RPC after error@618477663) -> (continuation of Deferred@1847772728 after retry RPC after error@1735514474) -> (continuation of Deferred@399520686 after retry RPC after error@1518530251) -> (continuation of Deferred@585756837 after retry RPC after error@1270531867) -> (continuation of Deferred@1566691457 after retry RPC after error@282630480) -> (continuation of Deferred@2111385398 after retry RPC after error@1615248777) -> (continuation of Deferred@871402712 after retry RPC after error@1731625455) -> (continuation of Deferred@1394778784 after retry RPC after error@115210168) -> (continuation of Deferred@838610423 after retry RPC after error@1654062046) -> (continuation of Deferred@619147918 after retry RPC after error@1299067293)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.6.8.252:39609,127.6.8.254:34321,127.6.8.253:41923) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.254:34321(127.6.8.254:34321): Connection refused: /127.6.8.254:34321,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.8.253:41923(127.6.8.253:41923): Connection refused: /127.6.8.253:41923
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.6.8.253:41923
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)