21:43:30.079 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
21:43:30.092 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskGzKGwb/test-tmp
21:43:30.096 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
21:43:30.100 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskGzKGwb/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
21:43:30.440 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
21:43:30.616 [DEBUG - main] (MiniKuduCluster.java:195) Response:
21:43:30.618 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
21:43:30.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:30.623557 374 mini_oidc.cc:152] Starting JWKS server
21:43:30.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:30.628324 374 webserver.cc:460] Webserver started at http://127.0.0.1:45607/ using document root <none> and password file <none>
21:43:30.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:30.628943 374 mini_oidc.cc:189] Starting OIDC Discovery server
21:43:30.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:30.631855 374 webserver.cc:460] Webserver started at http://127.0.0.1:34115/ using document root <none> and password file <none>
21:43:30.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-26T21:43:30Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
21:43:30.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-26T21:43:30Z Disabled control of system clock
21:43:30.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:30.690342 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskGzKGwb/build/tsan/bin/kudu
21:43:30.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskGzKGwb/build/tsan/bin/kudu
21:43:30.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/master-0/wal
21:43:30.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/master-0/data
21:43:30.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:43:30.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:43:30.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:43:30.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:43:30.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:43:30.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:43:30.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/master-0/logs
21:43:30.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/master-0/data/info.pb
21:43:30.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:43:30.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:43:30.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:43:30.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:43:30.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:43:30.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:43:30.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:43:30.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:43:30.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:43:30.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:43:30.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:43:30.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:43:30.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.190:42211
21:43:30.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.190
21:43:30.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:43:30.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:35563
21:43:30.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:43:30.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:43:30.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:43:30.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:43:30.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:42211,127.0.93.189:41305,127.0.93.188:33061
21:43:30.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:43:30.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45607/jwks.json with env {}
21:43:31.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:31.012517 386 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:43:31.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:31.013094 386 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:43:31.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:31.013612 386 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:43:31.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:31.046217 386 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:43:31.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:31.046533 386 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:43:31.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:31.046723 386 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:43:31.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:31.046898 386 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:43:31.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:31.047106 386 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:43:31.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:31.047289 386 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45607/jwks.json
21:43:31.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:31.092279 386 master_runner.cc:386] Master server non-default flags:
21:43:31.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:43:31.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:35563
21:43:31.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:43:31.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:43:31.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/master-0/data
21:43:31.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/master-0/wal
21:43:31.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:43:31.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:42211,127.0.93.189:41305,127.0.93.188:33061
21:43:31.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:43:31.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:43:31.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:43:31.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.190:42211
21:43:31.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:43:31.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:43:31.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:43:31.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45607/jwks.json
21:43:31.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:43:31.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:43:31.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/master-0/data/info.pb
21:43:31.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.190
21:43:31.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:43:31.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:43:31.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:43:31.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:43:31.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:43:31.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:43:31.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/master-0/logs
21:43:31.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:43:31.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:43:31.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:43:31.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:43:31.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
21:43:31.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
21:43:31.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:43:31.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 20:50:16 UTC on ccycloud.quasar-khhagj.root.comops.site
21:43:31.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4674730
21:43:31.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:43:31.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:31.093569 386 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:43:31.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:31.095376 386 file_cache.cc:493] Constructed file cache file cache with capacity 419430
21:43:31.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:31.123888 393 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:43:32.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:31.123878 392 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:43:32.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:31.125821 395 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:43:32.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:32.256784 394 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
21:43:32.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:32.256873 386 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:43:32.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:32.263027 386 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:43:32.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:32.266402 386 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:43:32.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:32.268028 386 hybrid_clock.cc:648] HybridClock initialized: now 1790459012267954 us; error 115 us; skew 500 ppm
21:43:32.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:32.269079 386 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:43:32.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:32.281270 386 webserver.cc:460] Webserver started at http://127.0.93.190:33833/ using document root <none> and password file <none>
21:43:32.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:32.282407 386 fs_manager.cc:359] Metadata directory not provided
21:43:32.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:32.282637 386 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:43:32.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:32.283245 386 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
21:43:32.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:32.288256 386 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/master-0/data/instance:
21:43:32.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9bc3212577d64be88d6ac70f394e4aef"
21:43:32.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 21:43:32 on dist-test-slave-b0jg"
21:43:32.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:32.289638 386 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/master-0/wal/instance:
21:43:32.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9bc3212577d64be88d6ac70f394e4aef"
21:43:32.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 21:43:32 on dist-test-slave-b0jg"
21:43:32.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:32.298477 386 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.010s sys 0.000s
21:43:32.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:32.305713 402 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:43:32.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:32.307346 386 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.001s sys 0.006s
21:43:32.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:32.307798 386 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/master-0/data,/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/master-0/wal
21:43:32.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9bc3212577d64be88d6ac70f394e4aef"
21:43:32.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 21:43:32 on dist-test-slave-b0jg"
21:43:32.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:32.308243 386 fs_report.cc:383] FS layout report
21:43:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:43:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/master-0/wal
21:43:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/master-0/wal
21:43:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/master-0/data/data
21:43:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:43:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:43:32.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:43:32.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:43:32.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:43:32.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:43:32.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:43:32.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:43:32.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:43:32.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:43:32.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:43:32.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:32.360419 386 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
21:43:32.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:32.362483 386 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:43:32.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:32.363049 386 kserver.cc:163] Server-wide thread pool size limit: 3276
21:43:32.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:32.453220 386 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.190:42211
21:43:32.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:32.453352 453 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.190:42211 every 8 connection(s)
21:43:32.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:32.456202 386 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/master-0/data/info.pb
21:43:32.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:32.463766 454 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:43:32.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:32.466178 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskGzKGwb/build/tsan/bin/kudu as pid 386
21:43:32.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:32.466635 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/master-0/wal/instance
21:43:32.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:32.471850 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskGzKGwb/build/tsan/bin/kudu
21:43:32.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskGzKGwb/build/tsan/bin/kudu
21:43:32.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/master-1/wal
21:43:32.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/master-1/data
21:43:32.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:43:32.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:43:32.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:43:32.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:43:32.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:43:32.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:43:32.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/master-1/logs
21:43:32.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/master-1/data/info.pb
21:43:32.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:43:32.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:43:32.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:43:32.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:43:32.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:43:32.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:43:32.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:43:32.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:43:32.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:43:32.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:43:32.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:43:32.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:43:32.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.189:41305
21:43:32.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.189
21:43:32.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:43:32.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:35563
21:43:32.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:43:32.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:43:32.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:43:32.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:43:32.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:42211,127.0.93.189:41305,127.0.93.188:33061
21:43:32.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:43:32.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45607/jwks.json with env {}
21:43:32.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:32.481132 454 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42211 } has no permanent_uuid. Determining permanent_uuid...
21:43:32.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:32.514964 454 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.189" port: 41305 } has no permanent_uuid. Determining permanent_uuid...
21:43:32.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:32.518266 403 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.93.189:41305: connect: Connection refused (error 111)
21:43:32.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:32.523980 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:41305: Network error: Client connection negotiation failed: client connection to 127.0.93.189:41305: connect: Connection refused (error 111)
21:43:32.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:32.573632 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 41305 } attempt: 1
21:43:32.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:32.577625 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:41305: Network error: Client connection negotiation failed: client connection to 127.0.93.189:41305: connect: Connection refused (error 111)
21:43:32.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:32.646225 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 41305 } attempt: 2
21:43:32.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:32.650481 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:41305: Network error: Client connection negotiation failed: client connection to 127.0.93.189:41305: connect: Connection refused (error 111)
21:43:32.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:32.742040 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 41305 } attempt: 3
21:43:32.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:32.746012 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:41305: Network error: Client connection negotiation failed: client connection to 127.0.93.189:41305: connect: Connection refused (error 111)
21:43:32.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:32.802386 456 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:43:32.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:32.803130 456 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:43:32.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:32.803709 456 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:43:32.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:32.839916 456 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:43:32.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:32.840272 456 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:43:32.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:32.840495 456 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:43:32.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:32.840677 456 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:43:32.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:32.840930 456 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:43:32.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:32.841166 456 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45607/jwks.json
21:43:32.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:32.880317 456 master_runner.cc:386] Master server non-default flags:
21:43:32.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:43:32.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:35563
21:43:32.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:43:32.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:43:32.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/master-1/data
21:43:32.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/master-1/wal
21:43:32.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:43:32.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:42211,127.0.93.189:41305,127.0.93.188:33061
21:43:32.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:43:32.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:43:32.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:43:32.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.189:41305
21:43:32.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:43:32.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:43:32.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:43:32.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45607/jwks.json
21:43:32.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:43:32.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:43:32.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/master-1/data/info.pb
21:43:32.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.189
21:43:32.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:43:32.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:43:32.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:43:32.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:43:32.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:43:32.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:43:32.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/master-1/logs
21:43:32.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:43:32.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:43:32.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:43:32.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:43:32.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
21:43:32.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
21:43:32.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:43:32.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 20:50:16 UTC on ccycloud.quasar-khhagj.root.comops.site
21:43:32.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4674730
21:43:32.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:43:32.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:32.881955 456 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:43:32.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:32.883978 456 file_cache.cc:493] Constructed file cache file cache with capacity 419430
21:43:32.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:32.889564 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 41305 } attempt: 4
21:43:32.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:32.894443 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:41305: Network error: Client connection negotiation failed: client connection to 127.0.93.189:41305: connect: Connection refused (error 111)
21:43:32.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:32.908143 466 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:43:33.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:33.193874 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 41305 } attempt: 5
21:43:33.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:33.198521 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:41305: Network error: Client connection negotiation failed: client connection to 127.0.93.189:41305: connect: Connection refused (error 111)
21:43:33.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:33.745994 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 41305 } attempt: 6
21:43:33.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:33.752789 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:41305: Network error: Client connection negotiation failed: client connection to 127.0.93.189:41305: connect: Connection refused (error 111)
21:43:34.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:32.908150 465 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:43:34.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:32.908298 468 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:43:34.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:32.909943 456 server_base.cc:1034] running on GCE node
21:43:34.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:34.058218 456 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:43:34.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:34.062091 456 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:43:34.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:34.063655 456 hybrid_clock.cc:648] HybridClock initialized: now 1790459014063576 us; error 97 us; skew 500 ppm
21:43:34.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:34.064680 456 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:43:34.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:34.073189 456 webserver.cc:460] Webserver started at http://127.0.93.189:36697/ using document root <none> and password file <none>
21:43:34.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:34.074662 456 fs_manager.cc:359] Metadata directory not provided
21:43:34.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:34.075042 456 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:43:34.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:34.075546 456 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
21:43:34.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:34.081450 456 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/master-1/data/instance:
21:43:34.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "87680c5ae6c549aea3a18e5e1ab59c83"
21:43:34.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 21:43:34 on dist-test-slave-b0jg"
21:43:34.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:34.082918 456 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/master-1/wal/instance:
21:43:34.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "87680c5ae6c549aea3a18e5e1ab59c83"
21:43:34.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 21:43:34 on dist-test-slave-b0jg"
21:43:34.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:34.092226 456 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.008s sys 0.000s
21:43:34.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:34.099860 477 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:43:34.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:34.101378 456 fs_manager.cc:725] Time spent opening block manager: real 0.006s user 0.001s sys 0.003s
21:43:34.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:34.101776 456 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/master-1/data,/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/master-1/wal
21:43:34.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "87680c5ae6c549aea3a18e5e1ab59c83"
21:43:34.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 21:43:34 on dist-test-slave-b0jg"
21:43:34.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:34.102142 456 fs_report.cc:383] FS layout report
21:43:34.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:43:34.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/master-1/wal
21:43:34.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/master-1/wal
21:43:34.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/master-1/data/data
21:43:34.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:43:34.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:43:34.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:43:34.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:43:34.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:43:34.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:43:34.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:43:34.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:43:34.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:43:34.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:43:34.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:43:34.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:34.141283 456 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
21:43:34.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:34.143728 456 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:43:34.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:34.144286 456 kserver.cc:163] Server-wide thread pool size limit: 3276
21:43:34.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:34.240296 456 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.189:41305
21:43:34.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:34.240417 528 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.189:41305 every 8 connection(s)
21:43:34.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:34.243304 456 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/master-1/data/info.pb
21:43:34.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:34.244055 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskGzKGwb/build/tsan/bin/kudu as pid 456
21:43:34.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:34.244450 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/master-1/wal/instance
21:43:34.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:34.246531 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskGzKGwb/build/tsan/bin/kudu
21:43:34.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskGzKGwb/build/tsan/bin/kudu
21:43:34.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/master-2/wal
21:43:34.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/master-2/data
21:43:34.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:43:34.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:43:34.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:43:34.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:43:34.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:43:34.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:43:34.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/master-2/logs
21:43:34.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/master-2/data/info.pb
21:43:34.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:43:34.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:43:34.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:43:34.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:43:34.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:43:34.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:43:34.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:43:34.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:43:34.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:43:34.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:43:34.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:43:34.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:43:34.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.188:33061
21:43:34.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.188
21:43:34.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:43:34.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:35563
21:43:34.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:43:34.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:43:34.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:43:34.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:43:34.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:42211,127.0.93.189:41305,127.0.93.188:33061
21:43:34.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:43:34.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45607/jwks.json with env {}
21:43:34.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:34.251734 529 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:43:34.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:34.264616 529 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42211 } has no permanent_uuid. Determining permanent_uuid...
21:43:34.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:34.292124 529 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.189" port: 41305 } has no permanent_uuid. Determining permanent_uuid...
21:43:34.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:34.312919 529 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33061 } has no permanent_uuid. Determining permanent_uuid...
21:43:34.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:34.315847 480 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.93.188:33061: connect: Connection refused (error 111)
21:43:34.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:34.321924 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:33061: Network error: Client connection negotiation failed: client connection to 127.0.93.188:33061: connect: Connection refused (error 111)
21:43:34.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:34.371503 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33061 } attempt: 1
21:43:34.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:34.376354 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:33061: Network error: Client connection negotiation failed: client connection to 127.0.93.188:33061: connect: Connection refused (error 111)
21:43:34.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:34.444775 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33061 } attempt: 2
21:43:34.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:34.451104 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:33061: Network error: Client connection negotiation failed: client connection to 127.0.93.188:33061: connect: Connection refused (error 111)
21:43:34.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:34.542543 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33061 } attempt: 3
21:43:34.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:34.546823 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:33061: Network error: Client connection negotiation failed: client connection to 127.0.93.188:33061: connect: Connection refused (error 111)
21:43:34.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:34.597165 531 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:43:34.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:34.597744 531 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:43:34.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:34.598353 531 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:43:34.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:34.629491 531 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:43:34.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:34.629794 531 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:43:34.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:34.629992 531 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:43:34.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:34.630215 531 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:43:34.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:34.630434 531 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:43:34.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:34.630568 531 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45607/jwks.json
21:43:34.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:34.665977 531 master_runner.cc:386] Master server non-default flags:
21:43:34.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:43:34.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:35563
21:43:34.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:43:34.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:43:34.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/master-2/data
21:43:34.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/master-2/wal
21:43:34.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:43:34.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:42211,127.0.93.189:41305,127.0.93.188:33061
21:43:34.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:43:34.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:43:34.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:43:34.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.188:33061
21:43:34.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:43:34.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:43:34.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:43:34.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45607/jwks.json
21:43:34.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:43:34.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:43:34.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/master-2/data/info.pb
21:43:34.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.188
21:43:34.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:43:34.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:43:34.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:43:34.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:43:34.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:43:34.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:43:34.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/master-2/logs
21:43:34.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:43:34.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:43:34.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:43:34.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:43:34.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
21:43:34.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
21:43:34.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:43:34.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 20:50:16 UTC on ccycloud.quasar-khhagj.root.comops.site
21:43:34.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4674730
21:43:34.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:43:34.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:34.667389 531 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:43:34.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:34.669374 531 file_cache.cc:493] Constructed file cache file cache with capacity 419430
21:43:34.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:34.690349 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33061 } attempt: 4
21:43:34.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:34.692864 541 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:43:34.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:34.695835 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:33061: Network error: Client connection negotiation failed: client connection to 127.0.93.188:33061: connect: Connection refused (error 111)
21:43:34.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:34.813441 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 41305 } attempt: 7
21:43:34.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:34.840786 454 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33061 } has no permanent_uuid. Determining permanent_uuid...
21:43:34.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:34.847265 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:33061: Network error: Client connection negotiation failed: client connection to 127.0.93.188:33061: connect: Connection refused (error 111)
21:43:34.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:34.905750 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33061 } attempt: 1
21:43:34.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:34.911780 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:33061: Network error: Client connection negotiation failed: client connection to 127.0.93.188:33061: connect: Connection refused (error 111)
21:43:34.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:34.993250 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33061 } attempt: 2
21:43:34.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:34.995289 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33061 } attempt: 5
21:43:35.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:34.999866 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:33061: Network error: Client connection negotiation failed: client connection to 127.0.93.188:33061: connect: Connection refused (error 111)
21:43:35.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:34.999910 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:33061: Network error: Client connection negotiation failed: client connection to 127.0.93.188:33061: connect: Connection refused (error 111)
21:43:35.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:35.085353 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33061 } attempt: 3
21:43:35.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:35.089936 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:33061: Network error: Client connection negotiation failed: client connection to 127.0.93.188:33061: connect: Connection refused (error 111)
21:43:35.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:35.230388 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33061 } attempt: 4
21:43:35.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:35.234900 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:33061: Network error: Client connection negotiation failed: client connection to 127.0.93.188:33061: connect: Connection refused (error 111)
21:43:35.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:35.518394 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33061 } attempt: 5
21:43:35.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:35.522511 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:33061: Network error: Client connection negotiation failed: client connection to 127.0.93.188:33061: connect: Connection refused (error 111)
21:43:35.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:35.547364 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33061 } attempt: 6
21:43:35.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:35.556252 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:33061: Network error: Client connection negotiation failed: client connection to 127.0.93.188:33061: connect: Connection refused (error 111)
21:43:35.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:34.692922 544 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:43:35.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:34.695070 542 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:43:35.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:34.697011 531 server_base.cc:1034] running on GCE node
21:43:35.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:35.856801 531 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:43:35.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:35.860575 531 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:43:35.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:35.862134 531 hybrid_clock.cc:648] HybridClock initialized: now 1790459015862078 us; error 74 us; skew 500 ppm
21:43:35.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:35.863206 531 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:43:35.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:35.871579 531 webserver.cc:460] Webserver started at http://127.0.93.188:33515/ using document root <none> and password file <none>
21:43:35.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:35.872634 531 fs_manager.cc:359] Metadata directory not provided
21:43:35.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:35.872911 531 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:43:35.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:35.873365 531 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
21:43:35.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:35.881696 531 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/master-2/data/instance:
21:43:35.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1ef0567afb574ae8a273ba7572ad56cd"
21:43:35.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 21:43:35 on dist-test-slave-b0jg"
21:43:35.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:35.883721 531 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/master-2/wal/instance:
21:43:35.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1ef0567afb574ae8a273ba7572ad56cd"
21:43:35.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 21:43:35 on dist-test-slave-b0jg"
21:43:35.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:35.896384 531 fs_manager.cc:691] Time spent creating directory manager: real 0.012s user 0.011s sys 0.001s
21:43:35.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:35.903198 556 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:43:35.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:35.904785 531 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.003s sys 0.001s
21:43:35.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:35.905153 531 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/master-2/data,/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/master-2/wal
21:43:35.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1ef0567afb574ae8a273ba7572ad56cd"
21:43:35.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 21:43:35 on dist-test-slave-b0jg"
21:43:35.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:35.905532 531 fs_report.cc:383] FS layout report
21:43:35.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:43:35.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/master-2/wal
21:43:35.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/master-2/wal
21:43:35.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/master-2/data/data
21:43:35.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:43:35.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:43:35.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:43:35.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:43:35.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:43:35.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:43:35.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:43:35.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:43:35.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:43:35.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:43:35.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:43:35.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:35.948450 531 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
21:43:35.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:35.950130 531 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:43:35.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:35.950615 531 kserver.cc:163] Server-wide thread pool size limit: 3276
21:43:36.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.049736 531 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.188:33061
21:43:36.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.049834 607 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.188:33061 every 8 connection(s)
21:43:36.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.053009 531 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/master-2/data/info.pb
21:43:36.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.060070 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskGzKGwb/build/tsan/bin/kudu as pid 531
21:43:36.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.060566 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/master-2/wal/instance
21:43:36.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.062567 608 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:43:36.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.075691 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33061 } attempt: 6
21:43:36.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.085091 608 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42211 } has no permanent_uuid. Determining permanent_uuid...
21:43:36.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.124686 608 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.189" port: 41305 } has no permanent_uuid. Determining permanent_uuid...
21:43:36.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.134146 454 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9bc3212577d64be88d6ac70f394e4aef: Bootstrap starting.
21:43:36.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.143815 454 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 9bc3212577d64be88d6ac70f394e4aef: Neither blocks nor log segments found. Creating new log.
21:43:36.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.144974 608 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33061 } has no permanent_uuid. Determining permanent_uuid...
21:43:36.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.146427 454 log.cc:824] T 00000000000000000000000000000000 P 9bc3212577d64be88d6ac70f394e4aef: Log is configured to *not* fsync() on all Append() calls
21:43:36.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.160758 454 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9bc3212577d64be88d6ac70f394e4aef: No bootstrap required, opened a new log
21:43:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.166778 608 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1ef0567afb574ae8a273ba7572ad56cd: Bootstrap starting.
21:43:36.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.176879 608 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 1ef0567afb574ae8a273ba7572ad56cd: Neither blocks nor log segments found. Creating new log.
21:43:36.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.180704 608 log.cc:824] T 00000000000000000000000000000000 P 1ef0567afb574ae8a273ba7572ad56cd: Log is configured to *not* fsync() on all Append() calls
21:43:36.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.186789 608 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1ef0567afb574ae8a273ba7572ad56cd: No bootstrap required, opened a new log
21:43:36.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.189318 454 raft_consensus.cc:348] T 00000000000000000000000000000000 P 9bc3212577d64be88d6ac70f394e4aef [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bc3212577d64be88d6ac70f394e4aef" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42211 } } peers { permanent_uuid: "87680c5ae6c549aea3a18e5e1ab59c83" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 41305 } } peers { permanent_uuid: "1ef0567afb574ae8a273ba7572ad56cd" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33061 } }
21:43:36.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.190834 454 raft_consensus.cc:374] T 00000000000000000000000000000000 P 9bc3212577d64be88d6ac70f394e4aef [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:43:36.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.191497 454 raft_consensus.cc:729] T 00000000000000000000000000000000 P 9bc3212577d64be88d6ac70f394e4aef [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9bc3212577d64be88d6ac70f394e4aef, State: Initialized, Role: FOLLOWER
21:43:36.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.192658 454 consensus_queue.cc:260] T 00000000000000000000000000000000 P 9bc3212577d64be88d6ac70f394e4aef [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bc3212577d64be88d6ac70f394e4aef" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42211 } } peers { permanent_uuid: "87680c5ae6c549aea3a18e5e1ab59c83" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 41305 } } peers { permanent_uuid: "1ef0567afb574ae8a273ba7572ad56cd" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33061 } }
21:43:36.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.196888 620 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9bc3212577d64be88d6ac70f394e4aef [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bc3212577d64be88d6ac70f394e4aef" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42211 } } peers { permanent_uuid: "87680c5ae6c549aea3a18e5e1ab59c83" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 41305 } } peers { permanent_uuid: "1ef0567afb574ae8a273ba7572ad56cd" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33061 } } }
21:43:36.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.199551 620 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9bc3212577d64be88d6ac70f394e4aef [sys.catalog]: This master's current role is: FOLLOWER
21:43:36.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.199122 454 sys_catalog.cc:564] T 00000000000000000000000000000000 P 9bc3212577d64be88d6ac70f394e4aef [sys.catalog]: configured and running, proceeding with master startup.
21:43:36.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.219938 608 raft_consensus.cc:348] T 00000000000000000000000000000000 P 1ef0567afb574ae8a273ba7572ad56cd [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bc3212577d64be88d6ac70f394e4aef" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42211 } } peers { permanent_uuid: "87680c5ae6c549aea3a18e5e1ab59c83" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 41305 } } peers { permanent_uuid: "1ef0567afb574ae8a273ba7572ad56cd" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33061 } }
21:43:36.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.221004 608 raft_consensus.cc:374] T 00000000000000000000000000000000 P 1ef0567afb574ae8a273ba7572ad56cd [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:43:36.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.221324 608 raft_consensus.cc:729] T 00000000000000000000000000000000 P 1ef0567afb574ae8a273ba7572ad56cd [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1ef0567afb574ae8a273ba7572ad56cd, State: Initialized, Role: FOLLOWER
21:43:36.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.222749 608 consensus_queue.cc:260] T 00000000000000000000000000000000 P 1ef0567afb574ae8a273ba7572ad56cd [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bc3212577d64be88d6ac70f394e4aef" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42211 } } peers { permanent_uuid: "87680c5ae6c549aea3a18e5e1ab59c83" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 41305 } } peers { permanent_uuid: "1ef0567afb574ae8a273ba7572ad56cd" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33061 } }
21:43:36.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.227541 629 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1ef0567afb574ae8a273ba7572ad56cd [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bc3212577d64be88d6ac70f394e4aef" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42211 } } peers { permanent_uuid: "87680c5ae6c549aea3a18e5e1ab59c83" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 41305 } } peers { permanent_uuid: "1ef0567afb574ae8a273ba7572ad56cd" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33061 } } }
21:43:36.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.229776 629 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1ef0567afb574ae8a273ba7572ad56cd [sys.catalog]: This master's current role is: FOLLOWER
21:43:36.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.230302 608 sys_catalog.cc:564] T 00000000000000000000000000000000 P 1ef0567afb574ae8a273ba7572ad56cd [sys.catalog]: configured and running, proceeding with master startup.
21:43:36.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:36.259616 632 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 9bc3212577d64be88d6ac70f394e4aef: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:43:36.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:36.260051 632 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:43:36.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.247640 386 master_runner.cc:186] Error getting master registration for 127.0.93.189:41305: OK, instance_id { permanent_uuid: "87680c5ae6c549aea3a18e5e1ab59c83" instance_seqno: 1790459014213992 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
21:43:36.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.287967 386 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:43:36.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:36.327730 644 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1ef0567afb574ae8a273ba7572ad56cd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:43:36.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:36.329059 644 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:43:36.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.332500 531 master_runner.cc:186] Error getting master registration for 127.0.93.189:41305: OK, instance_id { permanent_uuid: "87680c5ae6c549aea3a18e5e1ab59c83" instance_seqno: 1790459014213992 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
21:43:36.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.363781 531 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:43:36.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:36.373694 408 tablet.cc:2369] T 00000000000000000000000000000000 P 9bc3212577d64be88d6ac70f394e4aef: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:43:36.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:36.458238 562 tablet.cc:2369] T 00000000000000000000000000000000 P 1ef0567afb574ae8a273ba7572ad56cd: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:43:36.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.611320 620 raft_consensus.cc:482] T 00000000000000000000000000000000 P 9bc3212577d64be88d6ac70f394e4aef [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:43:36.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.611976 620 raft_consensus.cc:504] T 00000000000000000000000000000000 P 9bc3212577d64be88d6ac70f394e4aef [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bc3212577d64be88d6ac70f394e4aef" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42211 } } peers { permanent_uuid: "87680c5ae6c549aea3a18e5e1ab59c83" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 41305 } } peers { permanent_uuid: "1ef0567afb574ae8a273ba7572ad56cd" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33061 } }
21:43:36.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.614774 620 leader_election.cc:288] T 00000000000000000000000000000000 P 9bc3212577d64be88d6ac70f394e4aef [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 87680c5ae6c549aea3a18e5e1ab59c83 (127.0.93.189:41305), 1ef0567afb574ae8a273ba7572ad56cd (127.0.93.188:33061)
21:43:36.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.616845 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33061 } attempt: 7
21:43:36.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.617535 583 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9bc3212577d64be88d6ac70f394e4aef" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1ef0567afb574ae8a273ba7572ad56cd" is_pre_election: true
21:43:36.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.619544 583 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 1ef0567afb574ae8a273ba7572ad56cd [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9bc3212577d64be88d6ac70f394e4aef in term 0.
21:43:36.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.623490 405 leader_election.cc:302] T 00000000000000000000000000000000 P 9bc3212577d64be88d6ac70f394e4aef [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1ef0567afb574ae8a273ba7572ad56cd, 9bc3212577d64be88d6ac70f394e4aef; no voters:
21:43:36.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.624950 620 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 9bc3212577d64be88d6ac70f394e4aef [term 0 FOLLOWER]: Leader pre-election won for term 1
21:43:36.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.625382 620 raft_consensus.cc:482] T 00000000000000000000000000000000 P 9bc3212577d64be88d6ac70f394e4aef [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:43:36.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.625691 620 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 9bc3212577d64be88d6ac70f394e4aef [term 0 FOLLOWER]: Advancing to term 1
21:43:36.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.648805 620 raft_consensus.cc:504] T 00000000000000000000000000000000 P 9bc3212577d64be88d6ac70f394e4aef [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bc3212577d64be88d6ac70f394e4aef" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42211 } } peers { permanent_uuid: "87680c5ae6c549aea3a18e5e1ab59c83" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 41305 } } peers { permanent_uuid: "1ef0567afb574ae8a273ba7572ad56cd" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33061 } }
21:43:36.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.656514 620 leader_election.cc:288] T 00000000000000000000000000000000 P 9bc3212577d64be88d6ac70f394e4aef [CANDIDATE]: Term 1 election: Requested vote from peers 87680c5ae6c549aea3a18e5e1ab59c83 (127.0.93.189:41305), 1ef0567afb574ae8a273ba7572ad56cd (127.0.93.188:33061)
21:43:36.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.664463 583 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9bc3212577d64be88d6ac70f394e4aef" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1ef0567afb574ae8a273ba7572ad56cd"
21:43:36.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.665186 583 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 1ef0567afb574ae8a273ba7572ad56cd [term 0 FOLLOWER]: Advancing to term 1
21:43:36.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.671466 529 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 87680c5ae6c549aea3a18e5e1ab59c83: Bootstrap starting.
21:43:36.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.673082 583 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 1ef0567afb574ae8a273ba7572ad56cd [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9bc3212577d64be88d6ac70f394e4aef in term 1.
21:43:36.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.674875 405 leader_election.cc:302] T 00000000000000000000000000000000 P 9bc3212577d64be88d6ac70f394e4aef [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1ef0567afb574ae8a273ba7572ad56cd, 9bc3212577d64be88d6ac70f394e4aef; no voters:
21:43:36.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.676227 620 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 9bc3212577d64be88d6ac70f394e4aef [term 1 FOLLOWER]: Leader election won for term 1
21:43:36.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.681845 529 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 87680c5ae6c549aea3a18e5e1ab59c83: Neither blocks nor log segments found. Creating new log.
21:43:36.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.684543 620 raft_consensus.cc:686] T 00000000000000000000000000000000 P 9bc3212577d64be88d6ac70f394e4aef [term 1 LEADER]: Becoming Leader. State: Replica: 9bc3212577d64be88d6ac70f394e4aef, State: Running, Role: LEADER
21:43:36.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.686223 529 log.cc:824] T 00000000000000000000000000000000 P 87680c5ae6c549aea3a18e5e1ab59c83: Log is configured to *not* fsync() on all Append() calls
21:43:36.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.685969 620 consensus_queue.cc:237] T 00000000000000000000000000000000 P 9bc3212577d64be88d6ac70f394e4aef [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bc3212577d64be88d6ac70f394e4aef" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42211 } } peers { permanent_uuid: "87680c5ae6c549aea3a18e5e1ab59c83" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 41305 } } peers { permanent_uuid: "1ef0567afb574ae8a273ba7572ad56cd" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33061 } }
21:43:36.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.706275 529 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 87680c5ae6c549aea3a18e5e1ab59c83: No bootstrap required, opened a new log
21:43:36.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.711458 647 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9bc3212577d64be88d6ac70f394e4aef [sys.catalog]: SysCatalogTable state changed. Reason: New leader 9bc3212577d64be88d6ac70f394e4aef. Latest consensus state: current_term: 1 leader_uuid: "9bc3212577d64be88d6ac70f394e4aef" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bc3212577d64be88d6ac70f394e4aef" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42211 } } peers { permanent_uuid: "87680c5ae6c549aea3a18e5e1ab59c83" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 41305 } } peers { permanent_uuid: "1ef0567afb574ae8a273ba7572ad56cd" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33061 } } }
21:43:36.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.712308 647 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9bc3212577d64be88d6ac70f394e4aef [sys.catalog]: This master's current role is: LEADER
21:43:36.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.724574 650 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
21:43:36.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.739936 650 catalog_manager.cc:1484] Initializing Kudu cluster ID...
21:43:36.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.778486 583 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 1ef0567afb574ae8a273ba7572ad56cd [term 1 FOLLOWER]: Refusing update from remote peer 9bc3212577d64be88d6ac70f394e4aef: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:43:36.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.780643 529 raft_consensus.cc:348] T 00000000000000000000000000000000 P 87680c5ae6c549aea3a18e5e1ab59c83 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bc3212577d64be88d6ac70f394e4aef" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42211 } } peers { permanent_uuid: "87680c5ae6c549aea3a18e5e1ab59c83" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 41305 } } peers { permanent_uuid: "1ef0567afb574ae8a273ba7572ad56cd" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33061 } }
21:43:36.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.782521 529 raft_consensus.cc:374] T 00000000000000000000000000000000 P 87680c5ae6c549aea3a18e5e1ab59c83 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:43:36.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.782840 529 raft_consensus.cc:729] T 00000000000000000000000000000000 P 87680c5ae6c549aea3a18e5e1ab59c83 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 87680c5ae6c549aea3a18e5e1ab59c83, State: Initialized, Role: FOLLOWER
21:43:36.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.783947 529 consensus_queue.cc:260] T 00000000000000000000000000000000 P 87680c5ae6c549aea3a18e5e1ab59c83 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bc3212577d64be88d6ac70f394e4aef" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42211 } } peers { permanent_uuid: "87680c5ae6c549aea3a18e5e1ab59c83" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 41305 } } peers { permanent_uuid: "1ef0567afb574ae8a273ba7572ad56cd" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33061 } }
21:43:36.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.792325 647 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 9bc3212577d64be88d6ac70f394e4aef [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ef0567afb574ae8a273ba7572ad56cd" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33061 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
21:43:36.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.796222 652 sys_catalog.cc:455] T 00000000000000000000000000000000 P 87680c5ae6c549aea3a18e5e1ab59c83 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bc3212577d64be88d6ac70f394e4aef" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42211 } } peers { permanent_uuid: "87680c5ae6c549aea3a18e5e1ab59c83" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 41305 } } peers { permanent_uuid: "1ef0567afb574ae8a273ba7572ad56cd" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33061 } } }
21:43:36.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.797199 652 sys_catalog.cc:458] T 00000000000000000000000000000000 P 87680c5ae6c549aea3a18e5e1ab59c83 [sys.catalog]: This master's current role is: FOLLOWER
21:43:36.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.800225 529 sys_catalog.cc:564] T 00000000000000000000000000000000 P 87680c5ae6c549aea3a18e5e1ab59c83 [sys.catalog]: configured and running, proceeding with master startup.
21:43:36.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.800412 504 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9bc3212577d64be88d6ac70f394e4aef" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "87680c5ae6c549aea3a18e5e1ab59c83" is_pre_election: true
21:43:36.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.801749 504 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 87680c5ae6c549aea3a18e5e1ab59c83 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9bc3212577d64be88d6ac70f394e4aef in term 0.
21:43:36.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.804632 503 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9bc3212577d64be88d6ac70f394e4aef" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "87680c5ae6c549aea3a18e5e1ab59c83"
21:43:36.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.805368 503 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 87680c5ae6c549aea3a18e5e1ab59c83 [term 0 FOLLOWER]: Advancing to term 1
21:43:36.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.817474 503 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 87680c5ae6c549aea3a18e5e1ab59c83 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9bc3212577d64be88d6ac70f394e4aef in term 1.
21:43:36.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.822695 504 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 87680c5ae6c549aea3a18e5e1ab59c83 [term 1 FOLLOWER]: Refusing update from remote peer 9bc3212577d64be88d6ac70f394e4aef: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:43:36.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.845561 647 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 9bc3212577d64be88d6ac70f394e4aef [LEADER]: Connected to new peer: Peer: permanent_uuid: "87680c5ae6c549aea3a18e5e1ab59c83" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 41305 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
21:43:36.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.894449 652 sys_catalog.cc:455] T 00000000000000000000000000000000 P 87680c5ae6c549aea3a18e5e1ab59c83 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 9bc3212577d64be88d6ac70f394e4aef. Latest consensus state: current_term: 1 leader_uuid: "9bc3212577d64be88d6ac70f394e4aef" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bc3212577d64be88d6ac70f394e4aef" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42211 } } peers { permanent_uuid: "87680c5ae6c549aea3a18e5e1ab59c83" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 41305 } } peers { permanent_uuid: "1ef0567afb574ae8a273ba7572ad56cd" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33061 } } }
21:43:36.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.895431 652 sys_catalog.cc:458] T 00000000000000000000000000000000 P 87680c5ae6c549aea3a18e5e1ab59c83 [sys.catalog]: This master's current role is: FOLLOWER
21:43:36.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:36.904871 483 tablet.cc:2369] T 00000000000000000000000000000000 P 87680c5ae6c549aea3a18e5e1ab59c83: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:43:36.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.906879 655 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1ef0567afb574ae8a273ba7572ad56cd [sys.catalog]: SysCatalogTable state changed. Reason: New leader 9bc3212577d64be88d6ac70f394e4aef. Latest consensus state: current_term: 1 leader_uuid: "9bc3212577d64be88d6ac70f394e4aef" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bc3212577d64be88d6ac70f394e4aef" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42211 } } peers { permanent_uuid: "87680c5ae6c549aea3a18e5e1ab59c83" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 41305 } } peers { permanent_uuid: "1ef0567afb574ae8a273ba7572ad56cd" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33061 } } }
21:43:36.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.912845 655 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1ef0567afb574ae8a273ba7572ad56cd [sys.catalog]: This master's current role is: FOLLOWER
21:43:36.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.924996 651 mvcc.cc:205] Tried to move back new op lower bound from 7333720132689981440 to 7333720132357849088. Current Snapshot: MvccSnapshot[applied={T|T < 7333720132689981440}]
21:43:36.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.937646 662 mvcc.cc:205] Tried to move back new op lower bound from 7333720132689981440 to 7333720132357849088. Current Snapshot: MvccSnapshot[applied={T|T < 7333720132689981440}]
21:43:36.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:36.943419 668 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 87680c5ae6c549aea3a18e5e1ab59c83: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:43:36.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:36.943747 668 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:43:36.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.960338 661 mvcc.cc:205] Tried to move back new op lower bound from 7333720132689981440 to 7333720132357849088. Current Snapshot: MvccSnapshot[applied={T|T < 7333720132689981440}]
21:43:36.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.965569 650 catalog_manager.cc:1347] Generated new cluster ID: 08a568fd7f844fcf8011961cfb381e57
21:43:36.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.965911 650 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
21:43:36.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.966717 652 sys_catalog.cc:455] T 00000000000000000000000000000000 P 87680c5ae6c549aea3a18e5e1ab59c83 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "9bc3212577d64be88d6ac70f394e4aef" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bc3212577d64be88d6ac70f394e4aef" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42211 } } peers { permanent_uuid: "87680c5ae6c549aea3a18e5e1ab59c83" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 41305 } } peers { permanent_uuid: "1ef0567afb574ae8a273ba7572ad56cd" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33061 } } }
21:43:36.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.967908 652 sys_catalog.cc:458] T 00000000000000000000000000000000 P 87680c5ae6c549aea3a18e5e1ab59c83 [sys.catalog]: This master's current role is: FOLLOWER
21:43:36.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.976977 647 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9bc3212577d64be88d6ac70f394e4aef [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "9bc3212577d64be88d6ac70f394e4aef" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bc3212577d64be88d6ac70f394e4aef" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42211 } } peers { permanent_uuid: "87680c5ae6c549aea3a18e5e1ab59c83" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 41305 } } peers { permanent_uuid: "1ef0567afb574ae8a273ba7572ad56cd" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33061 } } }
21:43:36.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.984874 647 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9bc3212577d64be88d6ac70f394e4aef [sys.catalog]: This master's current role is: LEADER
21:43:37.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:36.990710 647 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9bc3212577d64be88d6ac70f394e4aef [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "9bc3212577d64be88d6ac70f394e4aef" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bc3212577d64be88d6ac70f394e4aef" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42211 } } peers { permanent_uuid: "87680c5ae6c549aea3a18e5e1ab59c83" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 41305 } } peers { permanent_uuid: "1ef0567afb574ae8a273ba7572ad56cd" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33061 } } }
21:43:37.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:37.003214 647 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9bc3212577d64be88d6ac70f394e4aef [sys.catalog]: This master's current role is: LEADER
21:43:37.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:37.021375 655 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1ef0567afb574ae8a273ba7572ad56cd [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "9bc3212577d64be88d6ac70f394e4aef" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9bc3212577d64be88d6ac70f394e4aef" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42211 } } peers { permanent_uuid: "87680c5ae6c549aea3a18e5e1ab59c83" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 41305 } } peers { permanent_uuid: "1ef0567afb574ae8a273ba7572ad56cd" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 33061 } } }
21:43:37.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:37.022260 655 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1ef0567afb574ae8a273ba7572ad56cd [sys.catalog]: This master's current role is: FOLLOWER
21:43:37.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:37.062041 650 catalog_manager.cc:1370] Generated new certificate authority record
21:43:37.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:37.065009 650 catalog_manager.cc:1504] Loading token signing keys...
21:43:37.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:37.084059 403 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.93.189:41305, user_credentials={real_user=slave}} blocked reactor thread for 49427us
21:43:37.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:37.124292 650 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 9bc3212577d64be88d6ac70f394e4aef: Generated new TSK 0
21:43:37.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:37.125743 650 catalog_manager.cc:1514] Initializing in-progress tserver states...
21:43:37.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:37.166455 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskGzKGwb/build/tsan/bin/kudu
21:43:37.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskGzKGwb/build/tsan/bin/kudu
21:43:37.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/ts-0/wal
21:43:37.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/ts-0/data
21:43:37.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:43:37.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:43:37.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:43:37.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:43:37.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:43:37.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:43:37.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/ts-0/logs
21:43:37.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/ts-0/data/info.pb
21:43:37.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:43:37.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:43:37.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:43:37.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:43:37.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:43:37.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:43:37.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:43:37.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:43:37.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:43:37.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:43:37.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.129:0
21:43:37.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.129
21:43:37.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.129
21:43:37.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:43:37.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:42211,127.0.93.189:41305,127.0.93.188:33061
21:43:37.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:35563
21:43:37.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:43:37.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:43:37.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:43:37.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:37.335474 644 catalog_manager.cc:1259] Loaded cluster ID: 08a568fd7f844fcf8011961cfb381e57
21:43:37.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:37.336290 644 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 1ef0567afb574ae8a273ba7572ad56cd: loading cluster ID for follower catalog manager: success
21:43:37.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:37.344861 644 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 1ef0567afb574ae8a273ba7572ad56cd: acquiring CA information for follower catalog manager: success
21:43:37.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:37.365311 644 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 1ef0567afb574ae8a273ba7572ad56cd: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:43:37.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:37.808218 675 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:43:37.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:37.809453 675 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:43:37.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:37.810467 675 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:43:37.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:37.871765 675 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:43:37.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:37.873869 675 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.93.129
21:43:37.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:37.939725 675 tablet_server_runner.cc:78] Tablet server non-default flags:
21:43:37.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:43:37.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:35563
21:43:37.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:43:37.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:43:37.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/ts-0/data
21:43:37.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/ts-0/wal
21:43:37.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:43:37.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:43:37.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.129:0
21:43:37.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:43:37.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:43:37.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:43:37.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/ts-0/data/info.pb
21:43:37.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.129
21:43:37.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:43:37.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:42211,127.0.93.189:41305,127.0.93.188:33061
21:43:37.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:43:37.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:43:37.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:43:37.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:43:37.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:43:37.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.129
21:43:37.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/ts-0/logs
21:43:37.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:43:37.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:43:37.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:43:37.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:43:37.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
21:43:37.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
21:43:37.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:43:37.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 20:50:16 UTC on ccycloud.quasar-khhagj.root.comops.site
21:43:37.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4674730
21:43:37.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:43:37.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:37.949313 675 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:43:37.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:37.949890 668 catalog_manager.cc:1259] Loaded cluster ID: 08a568fd7f844fcf8011961cfb381e57
21:43:37.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:37.950167 668 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 87680c5ae6c549aea3a18e5e1ab59c83: loading cluster ID for follower catalog manager: success
21:43:37.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:37.953087 675 file_cache.cc:493] Constructed file cache file cache with capacity 419430
21:43:37.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:37.959756 668 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 87680c5ae6c549aea3a18e5e1ab59c83: acquiring CA information for follower catalog manager: success
21:43:37.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:37.981957 668 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 87680c5ae6c549aea3a18e5e1ab59c83: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:43:38.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:38.010071 682 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:43:40.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:39.380746 680 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 675
21:43:40.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:40.477349 675 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 2.470s user 0.517s sys 0.823s
21:43:40.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:40.477739 675 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 2.471s user 0.517s sys 0.823s
21:43:40.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:38.013381 681 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:43:40.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:40.481267 685 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:43:40.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:40.483377 683 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2471 milliseconds
21:43:40.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:40.483408 675 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:43:40.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:40.484793 675 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:43:40.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:40.487299 675 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:43:40.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:40.488682 675 hybrid_clock.cc:648] HybridClock initialized: now 1790459020488643 us; error 61 us; skew 500 ppm
21:43:40.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:40.489625 675 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:43:40.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:40.501184 675 webserver.cc:460] Webserver started at http://127.0.93.129:39121/ using document root <none> and password file <none>
21:43:40.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:40.502375 675 fs_manager.cc:359] Metadata directory not provided
21:43:40.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:40.502630 675 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:43:40.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:40.503226 675 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
21:43:40.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:40.508771 675 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/ts-0/data/instance:
21:43:40.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cb9270541ca8410dbd45d12a86827d71"
21:43:40.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 21:43:40 on dist-test-slave-b0jg"
21:43:40.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:40.510205 675 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/ts-0/wal/instance:
21:43:40.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cb9270541ca8410dbd45d12a86827d71"
21:43:40.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 21:43:40 on dist-test-slave-b0jg"
21:43:40.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:40.519414 675 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.010s sys 0.001s
21:43:40.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:40.526366 691 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:43:40.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:40.529049 675 fs_manager.cc:725] Time spent opening block manager: real 0.006s user 0.003s sys 0.000s
21:43:40.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:40.529404 675 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/ts-0/data,/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/ts-0/wal
21:43:40.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cb9270541ca8410dbd45d12a86827d71"
21:43:40.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 21:43:40 on dist-test-slave-b0jg"
21:43:40.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:40.529773 675 fs_report.cc:383] FS layout report
21:43:40.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:43:40.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/ts-0/wal
21:43:40.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/ts-0/wal
21:43:40.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/ts-0/data/data
21:43:40.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:43:40.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:43:40.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:43:40.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:43:40.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:43:40.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:43:40.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:43:40.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:43:40.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:43:40.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:43:40.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:43:40.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:40.571703 675 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
21:43:40.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:40.573536 675 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:43:40.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:40.574410 675 kserver.cc:163] Server-wide thread pool size limit: 3276
21:43:40.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:40.577562 675 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
21:43:40.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:40.582613 675 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:43:40.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:40.582854 675 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
21:43:40.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:40.583132 675 ts_tablet_manager.cc:610] Registered 0 tablets
21:43:40.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:40.583285 675 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
21:43:40.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:40.770702 675 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.129:34093
21:43:40.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:40.770810 804 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.129:34093 every 8 connection(s)
21:43:40.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:40.774384 675 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/ts-0/data/info.pb
21:43:40.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:40.777257 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskGzKGwb/build/tsan/bin/kudu as pid 675
21:43:40.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:40.778053 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/ts-0/wal/instance
21:43:40.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:40.802980 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskGzKGwb/build/tsan/bin/kudu
21:43:40.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskGzKGwb/build/tsan/bin/kudu
21:43:40.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/ts-1/wal
21:43:40.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/ts-1/data
21:43:40.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:43:40.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:43:40.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:43:40.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:43:40.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:43:40.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:43:40.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/ts-1/logs
21:43:40.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/ts-1/data/info.pb
21:43:40.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:43:40.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:43:40.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:43:40.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:43:40.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:43:40.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:43:40.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:43:40.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:43:40.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:43:40.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:43:40.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.130:0
21:43:40.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.130
21:43:40.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.130
21:43:40.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:43:40.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:42211,127.0.93.189:41305,127.0.93.188:33061
21:43:40.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:35563
21:43:40.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:43:40.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:43:40.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:43:40.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:40.834975 806 heartbeater.cc:347] Connected to a master server at 127.0.93.189:41305
21:43:40.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:40.835577 806 heartbeater.cc:464] Registering TS with master...
21:43:40.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:40.836927 806 heartbeater.cc:511] Master 127.0.93.189:41305 requested a full tablet report, sending...
21:43:40.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:40.837354 805 heartbeater.cc:347] Connected to a master server at 127.0.93.188:33061
21:43:40.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:40.837703 805 heartbeater.cc:464] Registering TS with master...
21:43:40.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:40.839326 805 heartbeater.cc:511] Master 127.0.93.188:33061 requested a full tablet report, sending...
21:43:40.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:40.841741 494 ts_manager.cc:194] Registered new tserver with Master: cb9270541ca8410dbd45d12a86827d71 (127.0.93.129:34093)
21:43:40.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:40.842631 573 ts_manager.cc:194] Registered new tserver with Master: cb9270541ca8410dbd45d12a86827d71 (127.0.93.129:34093)
21:43:40.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:40.849257 807 heartbeater.cc:347] Connected to a master server at 127.0.93.190:42211
21:43:40.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:40.849766 807 heartbeater.cc:464] Registering TS with master...
21:43:40.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:40.850746 807 heartbeater.cc:511] Master 127.0.93.190:42211 requested a full tablet report, sending...
21:43:40.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:40.853654 419 ts_manager.cc:194] Registered new tserver with Master: cb9270541ca8410dbd45d12a86827d71 (127.0.93.129:34093)
21:43:40.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:40.856143 419 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.93.129:49153
21:43:41.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:41.180320 815 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:43:41.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:41.180891 815 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:43:41.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:41.181351 815 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:43:41.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:41.214428 815 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:43:41.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:41.215464 815 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.93.130
21:43:41.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:41.255640 815 tablet_server_runner.cc:78] Tablet server non-default flags:
21:43:41.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:43:41.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:35563
21:43:41.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:43:41.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:43:41.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/ts-1/data
21:43:41.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/ts-1/wal
21:43:41.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:43:41.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:43:41.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.130:0
21:43:41.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:43:41.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:43:41.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:43:41.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/ts-1/data/info.pb
21:43:41.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.130
21:43:41.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:43:41.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:42211,127.0.93.189:41305,127.0.93.188:33061
21:43:41.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:43:41.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:43:41.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:43:41.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:43:41.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:43:41.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.130
21:43:41.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/ts-1/logs
21:43:41.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:43:41.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:43:41.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:43:41.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:43:41.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
21:43:41.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
21:43:41.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:43:41.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 20:50:16 UTC on ccycloud.quasar-khhagj.root.comops.site
21:43:41.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4674730
21:43:41.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:43:41.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:41.257126 815 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:43:41.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:41.259302 815 file_cache.cc:493] Constructed file cache file cache with capacity 419430
21:43:41.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:41.281424 823 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:43:41.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:41.862329 807 heartbeater.cc:503] Master 127.0.93.190:42211 was elected leader, sending a full tablet report...
21:43:42.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:41.281437 825 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:43:42.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:41.282886 815 server_base.cc:1034] running on GCE node
21:43:42.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:41.282171 822 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:43:42.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:42.407399 815 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:43:42.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:42.410043 815 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:43:42.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:42.413705 815 hybrid_clock.cc:648] HybridClock initialized: now 1790459022413599 us; error 89 us; skew 500 ppm
21:43:42.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:42.414798 815 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:43:42.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:42.423126 815 webserver.cc:460] Webserver started at http://127.0.93.130:45525/ using document root <none> and password file <none>
21:43:42.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:42.424216 815 fs_manager.cc:359] Metadata directory not provided
21:43:42.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:42.424479 815 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:43:42.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:42.424969 815 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
21:43:42.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:42.431229 815 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/ts-1/data/instance:
21:43:42.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8a13e827785e42d4a8274557eb7c53d8"
21:43:42.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 21:43:42 on dist-test-slave-b0jg"
21:43:42.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:42.433501 815 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/ts-1/wal/instance:
21:43:42.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8a13e827785e42d4a8274557eb7c53d8"
21:43:42.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 21:43:42 on dist-test-slave-b0jg"
21:43:42.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:42.441851 815 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.009s sys 0.000s
21:43:42.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:42.449488 832 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:43:42.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:42.451092 815 fs_manager.cc:725] Time spent opening block manager: real 0.006s user 0.003s sys 0.003s
21:43:42.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:42.451494 815 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/ts-1/data,/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/ts-1/wal
21:43:42.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8a13e827785e42d4a8274557eb7c53d8"
21:43:42.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 21:43:42 on dist-test-slave-b0jg"
21:43:42.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:42.451855 815 fs_report.cc:383] FS layout report
21:43:42.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:43:42.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/ts-1/wal
21:43:42.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/ts-1/wal
21:43:42.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/ts-1/data/data
21:43:42.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:43:42.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:43:42.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:43:42.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:43:42.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:43:42.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:43:42.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:43:42.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:43:42.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:43:42.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:43:42.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:43:42.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:42.493917 815 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
21:43:42.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:42.496528 815 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:43:42.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:42.497043 815 kserver.cc:163] Server-wide thread pool size limit: 3276
21:43:42.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:42.499859 815 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
21:43:42.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:42.505378 815 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:43:42.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:42.505609 815 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
21:43:42.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:42.505816 815 ts_tablet_manager.cc:610] Registered 0 tablets
21:43:42.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:42.505947 815 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
21:43:42.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:42.698424 815 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.130:40369
21:43:42.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:42.698602 944 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.130:40369 every 8 connection(s)
21:43:42.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:42.701386 815 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/ts-1/data/info.pb
21:43:42.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:42.704667 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskGzKGwb/build/tsan/bin/kudu as pid 815
21:43:42.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:42.705132 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/ts-1/wal/instance
21:43:42.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:42.714879 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskGzKGwb/build/tsan/bin/kudu
21:43:42.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskGzKGwb/build/tsan/bin/kudu
21:43:42.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/ts-2/wal
21:43:42.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/ts-2/data
21:43:42.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:43:42.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:43:42.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:43:42.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:43:42.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:43:42.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:43:42.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/ts-2/logs
21:43:42.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/ts-2/data/info.pb
21:43:42.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:43:42.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:43:42.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:43:42.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:43:42.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:43:42.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:43:42.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:43:42.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:43:42.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:43:42.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:43:42.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.131:0
21:43:42.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.131
21:43:42.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.131
21:43:42.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:43:42.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:42211,127.0.93.189:41305,127.0.93.188:33061
21:43:42.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:35563
21:43:42.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:43:42.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:43:42.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:43:42.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:42.755980 946 heartbeater.cc:347] Connected to a master server at 127.0.93.189:41305
21:43:42.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:42.756531 946 heartbeater.cc:464] Registering TS with master...
21:43:42.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:42.757959 945 heartbeater.cc:347] Connected to a master server at 127.0.93.188:33061
21:43:42.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:42.757921 946 heartbeater.cc:511] Master 127.0.93.189:41305 requested a full tablet report, sending...
21:43:42.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:42.758394 945 heartbeater.cc:464] Registering TS with master...
21:43:42.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:42.759411 945 heartbeater.cc:511] Master 127.0.93.188:33061 requested a full tablet report, sending...
21:43:42.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:42.760139 947 heartbeater.cc:347] Connected to a master server at 127.0.93.190:42211
21:43:42.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:42.760553 947 heartbeater.cc:464] Registering TS with master...
21:43:42.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:42.761344 947 heartbeater.cc:511] Master 127.0.93.190:42211 requested a full tablet report, sending...
21:43:42.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:42.761477 494 ts_manager.cc:194] Registered new tserver with Master: 8a13e827785e42d4a8274557eb7c53d8 (127.0.93.130:40369)
21:43:42.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:42.763700 419 ts_manager.cc:194] Registered new tserver with Master: 8a13e827785e42d4a8274557eb7c53d8 (127.0.93.130:40369)
21:43:42.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:42.763972 573 ts_manager.cc:194] Registered new tserver with Master: 8a13e827785e42d4a8274557eb7c53d8 (127.0.93.130:40369)
21:43:42.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:42.765645 419 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.93.130:47015
21:43:43.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:43.103112 953 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:43:43.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:43.103700 953 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:43:43.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:43.104187 953 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:43:43.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:43.139365 953 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:43:43.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:43.140287 953 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.93.131
21:43:43.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:43.178813 953 tablet_server_runner.cc:78] Tablet server non-default flags:
21:43:43.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:43:43.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:35563
21:43:43.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:43:43.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:43:43.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/ts-2/data
21:43:43.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/ts-2/wal
21:43:43.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:43:43.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:43:43.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.131:0
21:43:43.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:43:43.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:43:43.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:43:43.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/ts-2/data/info.pb
21:43:43.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.131
21:43:43.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:43:43.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:42211,127.0.93.189:41305,127.0.93.188:33061
21:43:43.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:43:43.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:43:43.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:43:43.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:43:43.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:43:43.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.131
21:43:43.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/ts-2/logs
21:43:43.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:43:43.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:43:43.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:43:43.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:43:43.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
21:43:43.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
21:43:43.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:43:43.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 20:50:16 UTC on ccycloud.quasar-khhagj.root.comops.site
21:43:43.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4674730
21:43:43.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:43:43.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:43.180475 953 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:43:43.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:43.182612 953 file_cache.cc:493] Constructed file cache file cache with capacity 419430
21:43:43.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:43.204087 965 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:43:43.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:43.771059 947 heartbeater.cc:503] Master 127.0.93.190:42211 was elected leader, sending a full tablet report...
21:43:44.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:43.204154 962 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:43:44.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:43.204540 963 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:43:44.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:43.206607 953 server_base.cc:1034] running on GCE node
21:43:44.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:44.358055 953 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:43:44.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:44.360853 953 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:43:44.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:44.362310 953 hybrid_clock.cc:648] HybridClock initialized: now 1790459024362258 us; error 66 us; skew 500 ppm
21:43:44.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:44.363561 953 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:43:44.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:44.377749 953 webserver.cc:460] Webserver started at http://127.0.93.131:38303/ using document root <none> and password file <none>
21:43:44.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:44.378906 953 fs_manager.cc:359] Metadata directory not provided
21:43:44.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:44.379197 953 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:43:44.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:44.379688 953 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
21:43:44.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:44.385313 953 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/ts-2/data/instance:
21:43:44.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2bb5673a99744fbaa8e560cb5c22575d"
21:43:44.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 21:43:44 on dist-test-slave-b0jg"
21:43:44.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:44.386673 953 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/ts-2/wal/instance:
21:43:44.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2bb5673a99744fbaa8e560cb5c22575d"
21:43:44.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 21:43:44 on dist-test-slave-b0jg"
21:43:44.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:44.395678 953 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.007s sys 0.000s
21:43:44.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:44.404156 973 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:43:44.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:44.405627 953 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.001s sys 0.003s
21:43:44.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:44.406055 953 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/ts-2/data,/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/ts-2/wal
21:43:44.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2bb5673a99744fbaa8e560cb5c22575d"
21:43:44.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 21:43:44 on dist-test-slave-b0jg"
21:43:44.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:44.406463 953 fs_report.cc:383] FS layout report
21:43:44.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:43:44.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/ts-2/wal
21:43:44.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/ts-2/wal
21:43:44.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/ts-2/data/data
21:43:44.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:43:44.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:43:44.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:43:44.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:43:44.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:43:44.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:43:44.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:43:44.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:43:44.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:43:44.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:43:44.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:43:44.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:44.457842 953 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
21:43:44.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:44.459791 953 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:43:44.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:44.460397 953 kserver.cc:163] Server-wide thread pool size limit: 3276
21:43:44.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:44.464327 953 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
21:43:44.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:44.469659 953 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:43:44.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:44.469928 953 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
21:43:44.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:44.470239 953 ts_tablet_manager.cc:610] Registered 0 tablets
21:43:44.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:44.470440 953 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
21:43:44.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:44.657743 953 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.131:42151
21:43:44.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:44.657863 1086 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.131:42151 every 8 connection(s)
21:43:44.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:44.660624 953 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/ts-2/data/info.pb
21:43:44.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:44.665115 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskGzKGwb/build/tsan/bin/kudu as pid 953
21:43:44.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:44.665608 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4190671158177658303/ts-2/wal/instance
21:43:44.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:44.714920 1087 heartbeater.cc:347] Connected to a master server at 127.0.93.188:33061
21:43:44.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:44.715543 1087 heartbeater.cc:464] Registering TS with master...
21:43:44.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:44.716907 1087 heartbeater.cc:511] Master 127.0.93.188:33061 requested a full tablet report, sending...
21:43:44.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:44.718173 1088 heartbeater.cc:347] Connected to a master server at 127.0.93.189:41305
21:43:44.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:44.719295 1088 heartbeater.cc:464] Registering TS with master...
21:43:44.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:44.720125 1088 heartbeater.cc:511] Master 127.0.93.189:41305 requested a full tablet report, sending...
21:43:44.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:44.721863 573 ts_manager.cc:194] Registered new tserver with Master: 2bb5673a99744fbaa8e560cb5c22575d (127.0.93.131:42151)
21:43:44.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:44.723699 1089 heartbeater.cc:347] Connected to a master server at 127.0.93.190:42211
21:43:44.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:44.723718 494 ts_manager.cc:194] Registered new tserver with Master: 2bb5673a99744fbaa8e560cb5c22575d (127.0.93.131:42151)
21:43:44.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:44.724002 1089 heartbeater.cc:464] Registering TS with master...
21:43:44.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:44.724830 1089 heartbeater.cc:511] Master 127.0.93.190:42211 requested a full tablet report, sending...
21:43:44.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:44.726707 419 ts_manager.cc:194] Registered new tserver with Master: 2bb5673a99744fbaa8e560cb5c22575d (127.0.93.131:42151)
21:43:44.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:44.728322 419 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.93.131:48213
21:43:44.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:44.744630 374 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
21:43:44.758 [DEBUG - main] (MiniKuduCluster.java:195) Response:
21:43:44.760 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
21:43:44.774 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.0.93.190"
port: 42211
}
bound_http_address {
host: "127.0.93.190"
port: 33833
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.0.93.189"
port: 41305
}
bound_http_address {
host: "127.0.93.189"
port: 36697
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.0.93.188"
port: 33061
}
bound_http_address {
host: "127.0.93.188"
port: 33515
}
}
}
21:43:44.782 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
21:43:44.787 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.0.93.129"
port: 34093
}
bound_http_address {
host: "127.0.93.129"
port: 39121
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.0.93.130"
port: 40369
}
bound_http_address {
host: "127.0.93.130"
port: 45525
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.0.93.131"
port: 42151
}
bound_http_address {
host: "127.0.93.131"
port: 38303
}
}
}
21:43:44.787 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
21:43:44.989 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
21:43:45.138 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskGzKGwb/test-tmp
21:43:45.139 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
21:43:45.139 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskGzKGwb/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
21:43:45.143 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
21:43:45.596 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:43:45.597 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
21:43:45.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:45.600651 1098 mini_oidc.cc:152] Starting JWKS server
21:43:45.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:45.606500 1098 webserver.cc:460] Webserver started at http://127.0.0.1:40455/ using document root <none> and password file <none>
21:43:45.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:45.607105 1098 mini_oidc.cc:189] Starting OIDC Discovery server
21:43:45.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:45.609002 1098 webserver.cc:460] Webserver started at http://127.0.0.1:36893/ using document root <none> and password file <none>
21:43:45.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
21:43:45.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/krb5kdc/principal' for realm 'KRBTEST.COM',
21:43:45.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
21:43:45.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 21:43:45 dist-test-slave-b0jg krb5kdc[1110](info): setting up network...
21:43:45.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
21:43:45.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 21:43:45 dist-test-slave-b0jg krb5kdc[1110](info): set up 2 sockets
21:43:45.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 21:43:45 dist-test-slave-b0jg krb5kdc[1110](info): commencing operation
21:43:45.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
21:43:45.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:45.732161 1089 heartbeater.cc:503] Master 127.0.93.190:42211 was elected leader, sending a full tablet report...
21:43:46.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:46.675542 1098 mini_kdc.cc:121] Time spent starting KDC: real 1.038s user 0.000s sys 0.026s
21:43:46.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
21:43:46.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
21:43:46.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
21:43:46.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
21:43:46.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
21:43:46.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
21:43:46.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
21:43:46.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
21:43:46.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
21:43:46.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
21:43:46.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/krb5kdc/test-user.keytab.
21:43:46.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/krb5kdc/test-user.keytab.
21:43:46.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 21:43:46 dist-test-slave-b0jg krb5kdc[1110](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790459026, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:43:46.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
21:43:46.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-26T21:43:46Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
21:43:46.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-26T21:43:46Z Disabled control of system clock
21:43:46.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.18.190@KRBTEST.COM; defaulting to no policy
21:43:46.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:43:46.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.18.190@KRBTEST.COM" created.
21:43:46.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:43:46.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.18.190 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/krb5kdc/kudu_127.1.18.190.keytab.
21:43:46.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.18.190 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/krb5kdc/kudu_127.1.18.190.keytab.
21:43:46.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:46.897676 1098 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskGzKGwb/build/tsan/bin/kudu
21:43:46.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskGzKGwb/build/tsan/bin/kudu
21:43:46.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-0/wal
21:43:46.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-0/data
21:43:46.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:43:46.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:43:46.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:43:46.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:43:46.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:43:46.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:43:46.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-0/logs
21:43:46.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-0/data/info.pb
21:43:46.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:43:46.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:43:46.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:43:46.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:43:46.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:43:46.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:43:46.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:43:46.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:43:46.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:43:46.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:43:46.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:43:46.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:43:46.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.18.190:44871
21:43:46.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.18.190
21:43:46.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:43:46.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/krb5kdc/kudu_127.1.18.190.keytab
21:43:46.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.18.190
21:43:46.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:43:46.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:43:46.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:43:46.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.18.148:42639
21:43:46.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:43:46.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:43:46.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:43:46.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:43:46.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079
21:43:46.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:43:46.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40455/jwks.json
21:43:46.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:43:47.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:47.276132 1124 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:43:47.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:47.277040 1124 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:43:47.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:47.277992 1124 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:43:47.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:47.311769 1124 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:43:47.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:47.312091 1124 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:43:47.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:47.312557 1124 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:43:47.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:47.312914 1124 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:43:47.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:47.313277 1124 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:43:47.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:47.313627 1124 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:43:47.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:47.313894 1124 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40455/jwks.json
21:43:47.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:47.359910 1124 master_runner.cc:386] Master server non-default flags:
21:43:47.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:43:47.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.18.148:42639
21:43:47.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:43:47.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:43:47.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-0/data
21:43:47.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-0/wal
21:43:47.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:43:47.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:43:47.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079
21:43:47.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/krb5kdc/kudu_127.1.18.190.keytab
21:43:47.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.18.190
21:43:47.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:43:47.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:43:47.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:43:47.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.18.190:44871
21:43:47.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:43:47.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:43:47.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:43:47.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40455/jwks.json
21:43:47.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:43:47.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:43:47.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:43:47.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:43:47.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:43:47.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-0/data/info.pb
21:43:47.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.18.190
21:43:47.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:43:47.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:43:47.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:43:47.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:43:47.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:43:47.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:43:47.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-0/logs
21:43:47.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:43:47.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:43:47.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:43:47.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:43:47.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
21:43:47.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
21:43:47.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:43:47.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 20:50:16 UTC on ccycloud.quasar-khhagj.root.comops.site
21:43:47.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4674730
21:43:47.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:43:47.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:47.361333 1124 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:43:47.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:47.362488 1124 file_cache.cc:493] Constructed file cache file cache with capacity 419430
21:43:47.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:47.385731 1131 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:43:48.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:47.386085 1130 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:43:48.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:47.385725 1133 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:43:48.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:47.388577 1124 server_base.cc:1034] running on GCE node
21:43:48.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:48.585772 1124 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:43:48.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:48.588989 1124 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:43:48.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:48.590457 1124 hybrid_clock.cc:648] HybridClock initialized: now 1790459028590406 us; error 67 us; skew 500 ppm
21:43:48.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:48.591504 1124 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:43:48.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 21:43:48 dist-test-slave-b0jg krb5kdc[1110](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790459028, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.18.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:43:48.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:48.598289 1124 init.cc:377] Logged in from keytab as kudu/127.1.18.190@KRBTEST.COM (short username kudu)
21:43:48.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:48.605036 1124 webserver.cc:460] Webserver started at http://127.1.18.190:35511/ using document root <none> and password file <none>
21:43:48.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:48.606173 1124 fs_manager.cc:359] Metadata directory not provided
21:43:48.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:48.606405 1124 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:43:48.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:48.606947 1124 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
21:43:48.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:48.612794 1124 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-0/data/instance:
21:43:48.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bf123b719a624f9b99cc92ce3a1533b8"
21:43:48.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 21:43:48 on dist-test-slave-b0jg"
21:43:48.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:48.614248 1124 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-0/wal/instance:
21:43:48.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bf123b719a624f9b99cc92ce3a1533b8"
21:43:48.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 21:43:48 on dist-test-slave-b0jg"
21:43:48.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:48.624768 1124 fs_manager.cc:691] Time spent creating directory manager: real 0.010s user 0.004s sys 0.004s
21:43:48.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:48.632452 1142 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:43:48.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:48.633775 1124 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.002s sys 0.003s
21:43:48.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:48.634198 1124 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-0/data,/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-0/wal
21:43:48.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bf123b719a624f9b99cc92ce3a1533b8"
21:43:48.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 21:43:48 on dist-test-slave-b0jg"
21:43:48.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:48.634581 1124 fs_report.cc:383] FS layout report
21:43:48.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:43:48.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-0/wal
21:43:48.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-0/wal
21:43:48.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-0/data/data
21:43:48.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:43:48.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:43:48.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:43:48.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:43:48.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:43:48.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:43:48.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:43:48.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:43:48.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:43:48.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:43:48.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:43:48.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:48.675746 1124 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
21:43:48.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:48.682583 1124 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:43:48.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:48.683132 1124 kserver.cc:163] Server-wide thread pool size limit: 3276
21:43:48.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:48.788800 1124 rpc_server.cc:307] RPC server started. Bound to: 127.1.18.190:44871
21:43:48.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:48.788977 1193 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.18.190:44871 every 8 connection(s)
21:43:48.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:48.793406 1124 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-0/data/info.pb
21:43:48.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:48.795074 1098 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskGzKGwb/build/tsan/bin/kudu as pid 1124
21:43:48.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:48.795624 1098 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-0/wal/instance
21:43:48.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:48.803359 1194 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:43:48.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.18.189@KRBTEST.COM; defaulting to no policy
21:43:48.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:43:48.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.18.189@KRBTEST.COM" created.
21:43:48.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:48.822755 1194 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.18.190" port: 44871 } has no permanent_uuid. Determining permanent_uuid...
21:43:48.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:43:48.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.18.189 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/krb5kdc/kudu_127.1.18.189.keytab.
21:43:48.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.18.189 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/krb5kdc/kudu_127.1.18.189.keytab.
21:43:48.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:48.854477 1098 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskGzKGwb/build/tsan/bin/kudu
21:43:48.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskGzKGwb/build/tsan/bin/kudu
21:43:48.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-1/wal
21:43:48.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-1/data
21:43:48.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:43:48.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:43:48.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:43:48.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:43:48.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:43:48.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:43:48.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-1/logs
21:43:48.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-1/data/info.pb
21:43:48.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:43:48.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:43:48.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:43:48.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:43:48.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:43:48.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:43:48.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:43:48.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:43:48.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:43:48.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:43:48.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:43:48.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:43:48.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.18.189:41451
21:43:48.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.18.189
21:43:48.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:43:48.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/krb5kdc/kudu_127.1.18.189.keytab
21:43:48.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.18.189
21:43:48.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:43:48.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:43:48.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:43:48.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.18.148:42639
21:43:48.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:43:48.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:43:48.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:43:48.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:43:48.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079
21:43:48.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:43:48.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40455/jwks.json
21:43:48.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:43:48.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 21:43:48 dist-test-slave-b0jg krb5kdc[1110](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790459028, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.18.190@KRBTEST.COM for kudu/127.1.18.190@KRBTEST.COM
21:43:48.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:48.874001 1194 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.18.189" port: 41451 } has no permanent_uuid. Determining permanent_uuid...
21:43:48.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:48.878436 1145 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.18.189:41451: connect: Connection refused (error 111)
21:43:48.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:48.884014 1194 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.189:41451: Network error: Client connection negotiation failed: client connection to 127.1.18.189:41451: connect: Connection refused (error 111)
21:43:48.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:48.933549 1194 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.189" port: 41451 } attempt: 1
21:43:48.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:48.937459 1194 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.189:41451: Network error: Client connection negotiation failed: client connection to 127.1.18.189:41451: connect: Connection refused (error 111)
21:43:49.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:49.006060 1194 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.189" port: 41451 } attempt: 2
21:43:49.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:49.011263 1194 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.189:41451: Network error: Client connection negotiation failed: client connection to 127.1.18.189:41451: connect: Connection refused (error 111)
21:43:49.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:49.102778 1194 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.189" port: 41451 } attempt: 3
21:43:49.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:49.108310 1194 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.189:41451: Network error: Client connection negotiation failed: client connection to 127.1.18.189:41451: connect: Connection refused (error 111)
21:43:49.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:49.246909 1200 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:43:49.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:49.248016 1200 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:43:49.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:49.248862 1200 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:43:49.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:49.251792 1194 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.189" port: 41451 } attempt: 4
21:43:49.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:49.256500 1194 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.189:41451: Network error: Client connection negotiation failed: client connection to 127.1.18.189:41451: connect: Connection refused (error 111)
21:43:49.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:49.287348 1200 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:43:49.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:49.287638 1200 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:43:49.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:49.287907 1200 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:43:49.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:49.288139 1200 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:43:49.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:49.288394 1200 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:43:49.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:49.288619 1200 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:43:49.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:49.288774 1200 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40455/jwks.json
21:43:49.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:49.331960 1200 master_runner.cc:386] Master server non-default flags:
21:43:49.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:43:49.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.18.148:42639
21:43:49.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:43:49.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:43:49.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-1/data
21:43:49.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-1/wal
21:43:49.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:43:49.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:43:49.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079
21:43:49.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/krb5kdc/kudu_127.1.18.189.keytab
21:43:49.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.18.189
21:43:49.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:43:49.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:43:49.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:43:49.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.18.189:41451
21:43:49.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:43:49.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:43:49.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:43:49.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40455/jwks.json
21:43:49.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:43:49.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:43:49.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:43:49.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:43:49.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:43:49.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-1/data/info.pb
21:43:49.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.18.189
21:43:49.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:43:49.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:43:49.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:43:49.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:43:49.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:43:49.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:43:49.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-1/logs
21:43:49.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:43:49.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:43:49.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:43:49.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:43:49.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
21:43:49.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
21:43:49.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:43:49.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 20:50:16 UTC on ccycloud.quasar-khhagj.root.comops.site
21:43:49.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4674730
21:43:49.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:43:49.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:49.333449 1200 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:43:49.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:49.334714 1200 file_cache.cc:493] Constructed file cache file cache with capacity 419430
21:43:49.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:49.358529 1210 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:43:49.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:49.411544 801 debug-util.cc:398] Leaking SignalData structure 0x7b08000af320 after lost signal to thread 676
21:43:49.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:49.413272 801 debug-util.cc:398] Leaking SignalData structure 0x7b08000bcd40 after lost signal to thread 804
21:43:49.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:49.556005 1194 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.189" port: 41451 } attempt: 5
21:43:49.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:49.561498 1194 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.189:41451: Network error: Client connection negotiation failed: client connection to 127.1.18.189:41451: connect: Connection refused (error 111)
21:43:50.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:50.109251 1194 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.189" port: 41451 } attempt: 6
21:43:50.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:50.119144 1194 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.189:41451: Network error: Client connection negotiation failed: client connection to 127.1.18.189:41451: connect: Connection refused (error 111)
21:43:50.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:49.358559 1208 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:43:50.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:49.358940 1207 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:43:50.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:50.587168 1209 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
21:43:50.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:50.587635 1200 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:43:50.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:50.592214 1200 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:43:50.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:50.596009 1200 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:43:50.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:50.597944 1200 hybrid_clock.cc:648] HybridClock initialized: now 1790459030597820 us; error 110 us; skew 500 ppm
21:43:50.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:50.599278 1200 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:43:50.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 21:43:50 dist-test-slave-b0jg krb5kdc[1110](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790459030, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.18.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:43:50.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:50.608503 1200 init.cc:377] Logged in from keytab as kudu/127.1.18.189@KRBTEST.COM (short username kudu)
21:43:50.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:50.616052 1200 webserver.cc:460] Webserver started at http://127.1.18.189:41195/ using document root <none> and password file <none>
21:43:50.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:50.617182 1200 fs_manager.cc:359] Metadata directory not provided
21:43:50.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:50.617398 1200 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:43:50.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:50.618054 1200 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
21:43:50.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:50.624347 1200 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-1/data/instance:
21:43:50.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d22a615f9b324248ab926b797decf072"
21:43:50.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 21:43:50 on dist-test-slave-b0jg"
21:43:50.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:50.626525 1200 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-1/wal/instance:
21:43:50.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d22a615f9b324248ab926b797decf072"
21:43:50.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 21:43:50 on dist-test-slave-b0jg"
21:43:50.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:50.636238 1200 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.007s sys 0.001s
21:43:50.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:50.644131 1221 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:43:50.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:50.645856 1200 fs_manager.cc:725] Time spent opening block manager: real 0.006s user 0.004s sys 0.000s
21:43:50.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:50.646310 1200 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-1/data,/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-1/wal
21:43:50.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d22a615f9b324248ab926b797decf072"
21:43:50.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 21:43:50 on dist-test-slave-b0jg"
21:43:50.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:50.646828 1200 fs_report.cc:383] FS layout report
21:43:50.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:43:50.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-1/wal
21:43:50.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-1/wal
21:43:50.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-1/data/data
21:43:50.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:43:50.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:43:50.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:43:50.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:43:50.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:43:50.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:43:50.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:43:50.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:43:50.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:43:50.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:43:50.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:43:50.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:50.703636 1200 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
21:43:50.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:50.713054 1200 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:43:50.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:50.714478 1200 kserver.cc:163] Server-wide thread pool size limit: 3276
21:43:50.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:50.866791 1200 rpc_server.cc:307] RPC server started. Bound to: 127.1.18.189:41451
21:43:50.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:50.866885 1272 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.18.189:41451 every 8 connection(s)
21:43:50.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:50.870497 1200 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-1/data/info.pb
21:43:50.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:50.873575 1098 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskGzKGwb/build/tsan/bin/kudu as pid 1200
21:43:50.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:50.874086 1098 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-1/wal/instance
21:43:50.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:50.879547 1273 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:43:50.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.18.188@KRBTEST.COM; defaulting to no policy
21:43:50.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:43:50.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.18.188@KRBTEST.COM" created.
21:43:50.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:50.903047 1273 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.18.190" port: 44871 } has no permanent_uuid. Determining permanent_uuid...
21:43:50.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:43:50.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.18.188 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/krb5kdc/kudu_127.1.18.188.keytab.
21:43:50.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.18.188 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/krb5kdc/kudu_127.1.18.188.keytab.
21:43:50.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:50.938455 1098 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskGzKGwb/build/tsan/bin/kudu
21:43:50.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskGzKGwb/build/tsan/bin/kudu
21:43:50.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-2/wal
21:43:50.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-2/data
21:43:50.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:43:50.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:43:50.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:43:50.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:43:50.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:43:50.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:43:50.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-2/logs
21:43:50.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-2/data/info.pb
21:43:50.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:43:50.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:43:50.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:43:50.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:43:50.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:43:50.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:43:50.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:43:50.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:43:50.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:43:50.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:43:50.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:43:50.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:43:50.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.18.188:35079
21:43:50.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.18.188
21:43:50.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:43:50.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/krb5kdc/kudu_127.1.18.188.keytab
21:43:50.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.18.188
21:43:50.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:43:50.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:43:50.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:43:50.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.18.148:42639
21:43:50.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:43:50.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:43:50.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:43:50.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:43:50.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079
21:43:50.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:43:50.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40455/jwks.json
21:43:50.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:43:50.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 21:43:50 dist-test-slave-b0jg krb5kdc[1110](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790459030, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.18.189@KRBTEST.COM for kudu/127.1.18.190@KRBTEST.COM
21:43:50.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:50.962093 1273 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.18.189" port: 41451 } has no permanent_uuid. Determining permanent_uuid...
21:43:50.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 21:43:50 dist-test-slave-b0jg krb5kdc[1110](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790459030, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.18.189@KRBTEST.COM for kudu/127.1.18.189@KRBTEST.COM
21:43:51.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:51.003538 1273 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.18.188" port: 35079 } has no permanent_uuid. Determining permanent_uuid...
21:43:51.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:51.007624 1225 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.18.188:35079: connect: Connection refused (error 111)
21:43:51.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:51.014473 1273 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.188:35079: Network error: Client connection negotiation failed: client connection to 127.1.18.188:35079: connect: Connection refused (error 111)
21:43:51.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:51.064157 1273 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.188" port: 35079 } attempt: 1
21:43:51.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:51.070710 1273 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.188:35079: Network error: Client connection negotiation failed: client connection to 127.1.18.188:35079: connect: Connection refused (error 111)
21:43:51.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:51.139569 1273 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.188" port: 35079 } attempt: 2
21:43:51.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:51.143745 1273 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.188:35079: Network error: Client connection negotiation failed: client connection to 127.1.18.188:35079: connect: Connection refused (error 111)
21:43:51.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:51.179705 1194 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.189" port: 41451 } attempt: 7
21:43:51.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 21:43:51 dist-test-slave-b0jg krb5kdc[1110](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790459028, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.18.190@KRBTEST.COM for kudu/127.1.18.189@KRBTEST.COM
21:43:51.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:51.205917 1194 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.18.188" port: 35079 } has no permanent_uuid. Determining permanent_uuid...
21:43:51.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:51.212167 1194 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.188:35079: Network error: Client connection negotiation failed: client connection to 127.1.18.188:35079: connect: Connection refused (error 111)
21:43:51.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:51.235123 1273 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.188" port: 35079 } attempt: 3
21:43:51.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:51.239327 1273 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.188:35079: Network error: Client connection negotiation failed: client connection to 127.1.18.188:35079: connect: Connection refused (error 111)
21:43:51.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:51.270655 1194 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.188" port: 35079 } attempt: 1
21:43:51.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:51.274948 1194 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.188:35079: Network error: Client connection negotiation failed: client connection to 127.1.18.188:35079: connect: Connection refused (error 111)
21:43:51.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:51.338521 1279 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:43:51.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:51.339305 1279 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:43:51.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:51.340278 1279 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:43:51.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:51.356426 1194 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.188" port: 35079 } attempt: 2
21:43:51.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:51.361793 1194 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.188:35079: Network error: Client connection negotiation failed: client connection to 127.1.18.188:35079: connect: Connection refused (error 111)
21:43:51.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:51.377508 1279 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:43:51.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:51.377779 1279 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:43:51.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:51.378072 1279 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:43:51.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:51.378276 1279 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:43:51.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:51.378479 1279 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:43:51.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:51.378670 1279 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:43:51.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:51.378901 1279 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40455/jwks.json
21:43:51.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:51.382822 1273 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.188" port: 35079 } attempt: 4
21:43:51.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:51.386559 1273 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.188:35079: Network error: Client connection negotiation failed: client connection to 127.1.18.188:35079: connect: Connection refused (error 111)
21:43:51.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:51.419909 1279 master_runner.cc:386] Master server non-default flags:
21:43:51.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:43:51.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.18.148:42639
21:43:51.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:43:51.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:43:51.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-2/data
21:43:51.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-2/wal
21:43:51.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:43:51.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:43:51.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079
21:43:51.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/krb5kdc/kudu_127.1.18.188.keytab
21:43:51.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.18.188
21:43:51.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:43:51.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:43:51.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:43:51.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.18.188:35079
21:43:51.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:43:51.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:43:51.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:43:51.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40455/jwks.json
21:43:51.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:43:51.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:43:51.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:43:51.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:43:51.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:43:51.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-2/data/info.pb
21:43:51.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.18.188
21:43:51.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:43:51.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:43:51.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:43:51.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:43:51.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:43:51.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:43:51.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-2/logs
21:43:51.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:43:51.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:43:51.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:43:51.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:43:51.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
21:43:51.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
21:43:51.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:43:51.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 20:50:16 UTC on ccycloud.quasar-khhagj.root.comops.site
21:43:51.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4674730
21:43:51.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:43:51.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:51.421391 1279 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:43:51.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:51.422613 1279 file_cache.cc:493] Constructed file cache file cache with capacity 419430
21:43:51.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:51.447711 1291 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:43:51.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:51.448604 1194 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.188" port: 35079 } attempt: 3
21:43:51.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:51.455148 1194 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.188:35079: Network error: Client connection negotiation failed: client connection to 127.1.18.188:35079: connect: Connection refused (error 111)
21:43:51.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:51.595680 1194 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.188" port: 35079 } attempt: 4
21:43:51.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:51.600063 1194 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.188:35079: Network error: Client connection negotiation failed: client connection to 127.1.18.188:35079: connect: Connection refused (error 111)
21:43:51.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:51.686010 1273 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.188" port: 35079 } attempt: 5
21:43:51.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:51.691541 1273 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.188:35079: Network error: Client connection negotiation failed: client connection to 127.1.18.188:35079: connect: Connection refused (error 111)
21:43:51.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:51.883700 1194 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.188" port: 35079 } attempt: 5
21:43:51.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:51.889695 1194 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.188:35079: Network error: Client connection negotiation failed: client connection to 127.1.18.188:35079: connect: Connection refused (error 111)
21:43:52.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:52.238963 1273 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.188" port: 35079 } attempt: 6
21:43:52.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:52.248437 1273 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.188:35079: Network error: Client connection negotiation failed: client connection to 127.1.18.188:35079: connect: Connection refused (error 111)
21:43:52.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:52.442276 1194 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.188" port: 35079 } attempt: 6
21:43:52.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:52.470707 1194 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.188:35079: Network error: Client connection negotiation failed: client connection to 127.1.18.188:35079: connect: Connection refused (error 111)
21:43:52.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:51.450627 1292 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:43:52.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:51.452943 1279 server_base.cc:1034] running on GCE node
21:43:52.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:51.447745 1294 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:43:52.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:52.644819 1279 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:43:52.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:52.647814 1279 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:43:52.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:52.649261 1279 hybrid_clock.cc:648] HybridClock initialized: now 1790459032649216 us; error 70 us; skew 500 ppm
21:43:52.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:52.650163 1279 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:43:52.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 21:43:52 dist-test-slave-b0jg krb5kdc[1110](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790459032, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.18.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:43:52.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:52.657580 1279 init.cc:377] Logged in from keytab as kudu/127.1.18.188@KRBTEST.COM (short username kudu)
21:43:52.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:52.664093 1279 webserver.cc:460] Webserver started at http://127.1.18.188:40443/ using document root <none> and password file <none>
21:43:52.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:52.665072 1279 fs_manager.cc:359] Metadata directory not provided
21:43:52.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:52.665273 1279 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:43:52.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:52.665694 1279 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
21:43:52.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:52.671257 1279 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-2/data/instance:
21:43:52.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2b5583981b4048d2929b1e83236e47b2"
21:43:52.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 21:43:52 on dist-test-slave-b0jg"
21:43:52.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:52.672453 1279 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-2/wal/instance:
21:43:52.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2b5583981b4048d2929b1e83236e47b2"
21:43:52.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 21:43:52 on dist-test-slave-b0jg"
21:43:52.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:52.681080 1279 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.005s sys 0.004s
21:43:52.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:52.686900 1306 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:43:52.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:52.688076 1279 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
21:43:52.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:52.688385 1279 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-2/data,/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-2/wal
21:43:52.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2b5583981b4048d2929b1e83236e47b2"
21:43:52.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 21:43:52 on dist-test-slave-b0jg"
21:43:52.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:52.688694 1279 fs_report.cc:383] FS layout report
21:43:52.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:43:52.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-2/wal
21:43:52.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-2/wal
21:43:52.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-2/data/data
21:43:52.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:43:52.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:43:52.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:43:52.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:43:52.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:43:52.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:43:52.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:43:52.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:43:52.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:43:52.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:43:52.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:43:52.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:52.737475 1279 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
21:43:52.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:52.745613 1279 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:43:52.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:52.746176 1279 kserver.cc:163] Server-wide thread pool size limit: 3276
21:43:52.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:52.844035 1279 rpc_server.cc:307] RPC server started. Bound to: 127.1.18.188:35079
21:43:52.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:52.844172 1358 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.18.188:35079 every 8 connection(s)
21:43:52.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:52.846797 1279 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-2/data/info.pb
21:43:52.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:52.853426 1098 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskGzKGwb/build/tsan/bin/kudu as pid 1279
21:43:52.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:52.853938 1098 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-2/wal/instance
21:43:52.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:52.854511 1359 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:43:52.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:52.874439 1359 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.18.190" port: 44871 } has no permanent_uuid. Determining permanent_uuid...
21:43:52.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 21:43:52 dist-test-slave-b0jg krb5kdc[1110](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790459026, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.18.188@KRBTEST.COM
21:43:52.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 21:43:52 dist-test-slave-b0jg krb5kdc[1110](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790459032, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.18.188@KRBTEST.COM for kudu/127.1.18.190@KRBTEST.COM
21:43:52.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:52.924482 1359 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.18.189" port: 41451 } has no permanent_uuid. Determining permanent_uuid...
21:43:52.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 21:43:52 dist-test-slave-b0jg krb5kdc[1110](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790459026, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.18.189@KRBTEST.COM
21:43:52.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 21:43:52 dist-test-slave-b0jg krb5kdc[1110](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790459032, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.18.188@KRBTEST.COM for kudu/127.1.18.189@KRBTEST.COM
21:43:52.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:52.959231 1359 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.18.188" port: 35079 } has no permanent_uuid. Determining permanent_uuid...
21:43:52.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 21:43:52 dist-test-slave-b0jg krb5kdc[1110](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790459026, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.18.190@KRBTEST.COM
21:43:52.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 21:43:52 dist-test-slave-b0jg krb5kdc[1110](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790459032, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.18.188@KRBTEST.COM for kudu/127.1.18.188@KRBTEST.COM
21:43:52.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:52.995823 1098 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
21:43:52.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:52.996428 1359 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2: Bootstrap starting.
21:43:53.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:53.002637 1359 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2: Neither blocks nor log segments found. Creating new log.
21:43:53.004 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:43:53.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:53.005143 1359 log.cc:824] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2: Log is configured to *not* fsync() on all Append() calls
21:43:53.005 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
21:43:53.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:53.012337 1359 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2: No bootstrap required, opened a new log
21:43:53.019 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
21:43:53.022 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
21:43:53.025 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.1.18.190"
port: 44871
}
bound_http_address {
host: "127.1.18.190"
port: 35511
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.1.18.189"
port: 41451
}
bound_http_address {
host: "127.1.18.189"
port: 41195
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.1.18.188"
port: 35079
}
bound_http_address {
host: "127.1.18.188"
port: 40443
}
}
}
21:43:53.026 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
21:43:53.026 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:43:53.026 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
21:43:53.028 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
21:43:53.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:53.046710 1359 raft_consensus.cc:348] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bf123b719a624f9b99cc92ce3a1533b8" member_type: VOTER last_known_addr { host: "127.1.18.190" port: 44871 } } peers { permanent_uuid: "d22a615f9b324248ab926b797decf072" member_type: VOTER last_known_addr { host: "127.1.18.189" port: 41451 } } peers { permanent_uuid: "2b5583981b4048d2929b1e83236e47b2" member_type: VOTER last_known_addr { host: "127.1.18.188" port: 35079 } }
21:43:53.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:53.047638 1359 raft_consensus.cc:374] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:43:53.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:53.047926 1359 raft_consensus.cc:729] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2b5583981b4048d2929b1e83236e47b2, State: Initialized, Role: FOLLOWER
21:43:53.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:53.048974 1359 consensus_queue.cc:260] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bf123b719a624f9b99cc92ce3a1533b8" member_type: VOTER last_known_addr { host: "127.1.18.190" port: 44871 } } peers { permanent_uuid: "d22a615f9b324248ab926b797decf072" member_type: VOTER last_known_addr { host: "127.1.18.189" port: 41451 } } peers { permanent_uuid: "2b5583981b4048d2929b1e83236e47b2" member_type: VOTER last_known_addr { host: "127.1.18.188" port: 35079 } }
21:43:53.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:53.055980 1369 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bf123b719a624f9b99cc92ce3a1533b8" member_type: VOTER last_known_addr { host: "127.1.18.190" port: 44871 } } peers { permanent_uuid: "d22a615f9b324248ab926b797decf072" member_type: VOTER last_known_addr { host: "127.1.18.189" port: 41451 } } peers { permanent_uuid: "2b5583981b4048d2929b1e83236e47b2" member_type: VOTER last_known_addr { host: "127.1.18.188" port: 35079 } } }
21:43:53.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:53.057359 1369 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2 [sys.catalog]: This master's current role is: FOLLOWER
21:43:53.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:53.059850 1359 sys_catalog.cc:564] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2 [sys.catalog]: configured and running, proceeding with master startup.
21:43:53.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 21:43:53 dist-test-slave-b0jg krb5kdc[1110](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790459033, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:43:53.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
21:43:53.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:53.081266 1369 raft_consensus.cc:482] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:43:53.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:53.081892 1369 raft_consensus.cc:504] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bf123b719a624f9b99cc92ce3a1533b8" member_type: VOTER last_known_addr { host: "127.1.18.190" port: 44871 } } peers { permanent_uuid: "d22a615f9b324248ab926b797decf072" member_type: VOTER last_known_addr { host: "127.1.18.189" port: 41451 } } peers { permanent_uuid: "2b5583981b4048d2929b1e83236e47b2" member_type: VOTER last_known_addr { host: "127.1.18.188" port: 35079 } }
21:43:53.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:53.086144 1369 leader_election.cc:288] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers bf123b719a624f9b99cc92ce3a1533b8 (127.1.18.190:44871), d22a615f9b324248ab926b797decf072 (127.1.18.189:41451)
21:43:53.094 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:43:53.097 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/krb5kdc/krb5cc
21:43:53.106 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
21:43:53.108 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/krb5kdc/krb5cc
21:43:53.110 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
21:43:53.114 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
21:43:53.118 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
21:43:53.118 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
21:43:53.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:53.141077 1382 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:43:53.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:53.141613 1382 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:43:53.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:53.127192 1279 master_runner.cc:186] Error getting master registration for 127.1.18.190:44871: OK, instance_id { permanent_uuid: "bf123b719a624f9b99cc92ce3a1533b8" instance_seqno: 1790459028755806 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
21:43:53.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:53.168696 1279 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:43:53.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 21:43:53 dist-test-slave-b0jg krb5kdc[1110](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790459033, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:43:53.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
21:43:53.180 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:43:53.180 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/krb5kdc/krb5cc
21:43:53.182 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
21:43:53.183 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.1.18.190:44871
21:43:53.186 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
21:43:53.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:53.186748 1098 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskGzKGwb/build/tsan/bin/kudu with pid 1124
21:43:53.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:53.214063 1307 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv got EOF from 127.1.18.190:44871 (error 108)
21:43:53.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:53.220247 1364 negotiation.cc:337] Failed RPC negotiation. Trace:
21:43:53.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0926 21:43:53.217936 (+ 0us) reactor.cc:625] Submitting negotiation task for client connection to 127.1.18.190:44871 (local address 127.0.0.1:35684)
21:43:53.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0926 21:43:53.218472 (+ 536us) negotiation.cc:107] Waiting for socket to connect
21:43:53.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0926 21:43:53.218728 (+ 256us) negotiation.cc:327] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.1.18.190:44871: connect: Connection reset by peer (error 104)
21:43:53.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"client-negotiator.queue_time_us":102}
21:43:53.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:53.225018 1307 leader_election.cc:334] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer bf123b719a624f9b99cc92ce3a1533b8 (127.1.18.190:44871): Network error: Client connection negotiation failed: client connection to 127.1.18.190:44871: connect: Connection reset by peer (error 104)
21:43:53.232 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:43:53.233 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.1.18.189:41451
21:43:53.233 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
21:43:53.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:53.234066 1098 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskGzKGwb/build/tsan/bin/kudu with pid 1200
21:43:53.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:53.255298 1312 tablet.cc:2369] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:43:53.269 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:43:53.269 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.1.18.188:35079
21:43:53.269 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
21:43:53.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:53.270283 1098 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskGzKGwb/build/tsan/bin/kudu with pid 1279
21:43:53.308 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:43:53.309 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.1.18.190:44871
21:43:53.311 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
21:43:53.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:53.313827 1098 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskGzKGwb/build/tsan/bin/kudu
21:43:53.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskGzKGwb/build/tsan/bin/kudu
21:43:53.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-0/wal
21:43:53.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-0/data
21:43:53.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:43:53.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:43:53.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:43:53.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:43:53.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:43:53.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:43:53.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-0/logs
21:43:53.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-0/data/info.pb
21:43:53.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:43:53.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:43:53.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:43:53.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:43:53.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:43:53.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:43:53.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:43:53.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:43:53.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:43:53.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:43:53.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:43:53.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:43:53.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.18.190:44871
21:43:53.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.18.190
21:43:53.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35511
21:43:53.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/krb5kdc/kudu_127.1.18.190.keytab
21:43:53.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.18.190
21:43:53.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:43:53.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:43:53.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:43:53.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.18.148:42639
21:43:53.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:43:53.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:43:53.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:43:53.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:43:53.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079
21:43:53.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:43:53.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40455/jwks.json
21:43:53.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:43:53.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:53.455631 941 debug-util.cc:398] Leaking SignalData structure 0x7b08000b89a0 after lost signal to thread 817
21:43:53.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:53.456844 941 debug-util.cc:398] Leaking SignalData structure 0x7b0800040080 after lost signal to thread 944
21:43:53.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:53.670931 1387 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:43:53.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:53.671720 1387 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:43:53.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:53.672473 1387 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:43:53.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:53.706792 1387 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:43:53.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:53.707122 1387 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:43:53.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:53.707531 1387 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:43:53.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:53.707842 1387 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:43:53.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:53.708143 1387 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:43:53.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:53.708443 1387 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:43:53.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:53.708680 1387 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40455/jwks.json
21:43:53.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:53.751348 1387 master_runner.cc:386] Master server non-default flags:
21:43:53.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:43:53.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.18.148:42639
21:43:53.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:43:53.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:43:53.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-0/data
21:43:53.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-0/wal
21:43:53.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:43:53.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:43:53.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079
21:43:53.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/krb5kdc/kudu_127.1.18.190.keytab
21:43:53.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.18.190
21:43:53.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:43:53.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:43:53.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:43:53.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.18.190:44871
21:43:53.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:43:53.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:43:53.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:43:53.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40455/jwks.json
21:43:53.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:43:53.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:43:53.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:43:53.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:43:53.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:43:53.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-0/data/info.pb
21:43:53.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.18.190
21:43:53.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35511
21:43:53.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:43:53.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:43:53.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:43:53.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:43:53.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:43:53.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-0/logs
21:43:53.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:43:53.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:43:53.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:43:53.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:43:53.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
21:43:53.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
21:43:53.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:43:53.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 20:50:16 UTC on ccycloud.quasar-khhagj.root.comops.site
21:43:53.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4674730
21:43:53.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:43:53.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:53.753121 1387 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:43:53.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:53.754679 1387 file_cache.cc:493] Constructed file cache file cache with capacity 419430
21:43:53.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:53.782028 1393 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:43:55.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:53.782292 1394 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:43:55.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:53.784085 1396 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:43:55.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:55.209218 1395 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1425 milliseconds
21:43:55.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:55.209367 1387 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:43:55.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:55.210783 1387 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:43:55.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:55.214227 1387 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:43:55.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:55.215716 1387 hybrid_clock.cc:648] HybridClock initialized: now 1790459035215672 us; error 72 us; skew 500 ppm
21:43:55.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:55.216830 1387 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:43:55.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 21:43:55 dist-test-slave-b0jg krb5kdc[1110](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790459035, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.18.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:43:55.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:55.224416 1387 init.cc:377] Logged in from keytab as kudu/127.1.18.190@KRBTEST.COM (short username kudu)
21:43:55.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:55.231743 1387 webserver.cc:460] Webserver started at http://127.1.18.190:35511/ using document root <none> and password file <none>
21:43:55.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:55.232970 1387 fs_manager.cc:359] Metadata directory not provided
21:43:55.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:55.233291 1387 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:43:55.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:55.243443 1387 fs_manager.cc:709] Time spent opening directory manager: real 0.007s user 0.007s sys 0.000s
21:43:55.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:55.250468 1405 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:43:55.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:55.252086 1387 fs_manager.cc:725] Time spent opening block manager: real 0.006s user 0.005s sys 0.000s
21:43:55.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:55.252503 1387 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-0/data,/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-0/wal
21:43:55.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bf123b719a624f9b99cc92ce3a1533b8"
21:43:55.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 21:43:48 on dist-test-slave-b0jg"
21:43:55.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:55.256152 1387 fs_report.cc:383] FS layout report
21:43:55.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:43:55.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-0/wal
21:43:55.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-0/wal
21:43:55.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-0/data/data
21:43:55.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:43:55.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:43:55.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:43:55.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:43:55.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:43:55.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:43:55.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:43:55.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:43:55.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:43:55.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:43:55.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:43:55.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:55.298470 1387 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
21:43:55.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:55.305739 1387 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:43:55.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:55.306396 1387 kserver.cc:163] Server-wide thread pool size limit: 3276
21:43:55.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:55.402390 1387 rpc_server.cc:307] RPC server started. Bound to: 127.1.18.190:44871
21:43:55.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:55.402582 1456 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.18.190:44871 every 8 connection(s)
21:43:55.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:55.405750 1387 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-0/data/info.pb
21:43:55.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:55.406627 1098 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskGzKGwb/build/tsan/bin/kudu as pid 1387
21:43:55.407 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:43:55.407 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.1.18.189:41451
21:43:55.408 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
21:43:55.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:55.410177 1098 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskGzKGwb/build/tsan/bin/kudu
21:43:55.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskGzKGwb/build/tsan/bin/kudu
21:43:55.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-1/wal
21:43:55.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-1/data
21:43:55.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:43:55.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:43:55.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:43:55.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:43:55.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:43:55.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:43:55.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-1/logs
21:43:55.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-1/data/info.pb
21:43:55.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:43:55.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:43:55.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:43:55.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:43:55.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:43:55.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:43:55.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:43:55.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:43:55.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:43:55.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:43:55.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:43:55.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:43:55.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.18.189:41451
21:43:55.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.18.189
21:43:55.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41195
21:43:55.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/krb5kdc/kudu_127.1.18.189.keytab
21:43:55.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.18.189
21:43:55.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:43:55.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:43:55.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:43:55.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.18.148:42639
21:43:55.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:43:55.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:43:55.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:43:55.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:43:55.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079
21:43:55.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:43:55.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40455/jwks.json
21:43:55.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:43:55.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:55.429836 1457 sys_catalog.cc:263] Verifying existing consensus state
21:43:55.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260926 21:43:55.431406 1457 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
21:43:55.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:55.432102 1387 master.cc:537] Master@127.1.18.190:44871 shutting down...
21:43:55.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:55.522279 1387 master.cc:552] Master@127.1.18.190:44871 shutdown complete.
21:43:55.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
21:43:55.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:55.806253 1459 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:43:55.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:55.806938 1459 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:43:55.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:55.807646 1459 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:43:55.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:55.844612 1459 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:43:55.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:55.844880 1459 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:43:55.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:55.845153 1459 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:43:55.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:55.845356 1459 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:43:55.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:55.845672 1459 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:43:55.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:55.846009 1459 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:43:55.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:55.846252 1459 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40455/jwks.json
21:43:55.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:55.886747 1459 master_runner.cc:386] Master server non-default flags:
21:43:55.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:43:55.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.18.148:42639
21:43:55.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:43:55.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:43:55.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-1/data
21:43:55.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-1/wal
21:43:55.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:43:55.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:43:55.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079
21:43:55.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/krb5kdc/kudu_127.1.18.189.keytab
21:43:55.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.18.189
21:43:55.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:43:55.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:43:55.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:43:55.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.18.189:41451
21:43:55.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:43:55.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:43:55.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:43:55.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40455/jwks.json
21:43:55.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:43:55.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:43:55.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:43:55.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:43:55.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:43:55.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-1/data/info.pb
21:43:55.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.18.189
21:43:55.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41195
21:43:55.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:43:55.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:43:55.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:43:55.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:43:55.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:43:55.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-1/logs
21:43:55.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:43:55.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:43:55.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:43:55.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:43:55.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
21:43:55.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
21:43:55.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:43:55.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 20:50:16 UTC on ccycloud.quasar-khhagj.root.comops.site
21:43:55.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4674730
21:43:55.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:43:55.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:55.888660 1459 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:43:55.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:55.890080 1459 file_cache.cc:493] Constructed file cache file cache with capacity 419430
21:43:55.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:55.912863 1469 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:43:57.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:55.912851 1467 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:43:57.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:55.915055 1459 server_base.cc:1034] running on GCE node
21:43:57.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:55.912851 1466 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:43:57.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:57.094132 1459 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:43:57.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:57.096925 1459 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:43:57.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:57.098345 1459 hybrid_clock.cc:648] HybridClock initialized: now 1790459037098284 us; error 93 us; skew 500 ppm
21:43:57.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:57.099310 1459 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:43:57.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 21:43:57 dist-test-slave-b0jg krb5kdc[1110](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790459037, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.18.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:43:57.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:57.110255 1459 init.cc:377] Logged in from keytab as kudu/127.1.18.189@KRBTEST.COM (short username kudu)
21:43:57.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:57.115764 1459 webserver.cc:460] Webserver started at http://127.1.18.189:41195/ using document root <none> and password file <none>
21:43:57.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:57.116883 1459 fs_manager.cc:359] Metadata directory not provided
21:43:57.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:57.117128 1459 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:43:57.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:57.126497 1459 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.006s sys 0.001s
21:43:57.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:57.131215 1477 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:43:57.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:57.132588 1459 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
21:43:57.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:57.132938 1459 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-1/data,/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-1/wal
21:43:57.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d22a615f9b324248ab926b797decf072"
21:43:57.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 21:43:50 on dist-test-slave-b0jg"
21:43:57.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:57.135388 1459 fs_report.cc:383] FS layout report
21:43:57.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:43:57.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-1/wal
21:43:57.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-1/wal
21:43:57.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-1/data/data
21:43:57.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:43:57.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:43:57.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:43:57.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:43:57.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:43:57.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:43:57.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:43:57.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:43:57.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:43:57.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:43:57.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:43:57.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:57.171444 1459 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
21:43:57.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:57.177449 1459 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:43:57.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:57.177929 1459 kserver.cc:163] Server-wide thread pool size limit: 3276
21:43:57.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:57.260298 1459 rpc_server.cc:307] RPC server started. Bound to: 127.1.18.189:41451
21:43:57.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:57.260497 1529 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.18.189:41451 every 8 connection(s)
21:43:57.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:57.263566 1459 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-1/data/info.pb
21:43:57.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:57.265316 1098 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskGzKGwb/build/tsan/bin/kudu as pid 1459
21:43:57.270 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:43:57.270 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.1.18.188:35079
21:43:57.270 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
21:43:57.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:57.272819 1098 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskGzKGwb/build/tsan/bin/kudu
21:43:57.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskGzKGwb/build/tsan/bin/kudu
21:43:57.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-2/wal
21:43:57.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-2/data
21:43:57.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:43:57.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:43:57.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:43:57.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:43:57.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:43:57.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:43:57.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-2/logs
21:43:57.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-2/data/info.pb
21:43:57.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:43:57.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:43:57.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:43:57.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:43:57.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:43:57.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:43:57.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:43:57.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:43:57.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:43:57.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:43:57.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:43:57.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:43:57.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.18.188:35079
21:43:57.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.18.188
21:43:57.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40443
21:43:57.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/krb5kdc/kudu_127.1.18.188.keytab
21:43:57.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.18.188
21:43:57.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:43:57.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:43:57.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:43:57.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.18.148:42639
21:43:57.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:43:57.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:43:57.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:43:57.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:43:57.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079
21:43:57.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:43:57.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40455/jwks.json
21:43:57.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:43:57.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:57.280786 1530 sys_catalog.cc:263] Verifying existing consensus state
21:43:57.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260926 21:43:57.281742 1530 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
21:43:57.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:57.282215 1459 master.cc:537] Master@127.1.18.189:41451 shutting down...
21:43:57.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:57.338905 1459 master.cc:552] Master@127.1.18.189:41451 shutdown complete.
21:43:57.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
21:43:57.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:57.625774 1532 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:43:57.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:57.626605 1532 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:43:57.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:57.627405 1532 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:43:57.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:57.661644 1532 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:43:57.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:57.661970 1532 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:43:57.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:57.662421 1532 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:43:57.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:57.662748 1532 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:43:57.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:57.663085 1532 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:43:57.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:57.663410 1532 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:43:57.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:57.663640 1532 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40455/jwks.json
21:43:57.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:57.705698 1532 master_runner.cc:386] Master server non-default flags:
21:43:57.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:43:57.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.18.148:42639
21:43:57.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:43:57.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:43:57.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-2/data
21:43:57.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-2/wal
21:43:57.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:43:57.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:43:57.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079
21:43:57.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/krb5kdc/kudu_127.1.18.188.keytab
21:43:57.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.18.188
21:43:57.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:43:57.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:43:57.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:43:57.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.18.188:35079
21:43:57.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:43:57.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:43:57.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:43:57.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40455/jwks.json
21:43:57.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:43:57.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:43:57.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:43:57.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:43:57.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:43:57.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-2/data/info.pb
21:43:57.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.18.188
21:43:57.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40443
21:43:57.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:43:57.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:43:57.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:43:57.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:43:57.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:43:57.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-2/logs
21:43:57.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:43:57.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:43:57.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:43:57.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:43:57.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
21:43:57.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
21:43:57.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:43:57.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 20:50:16 UTC on ccycloud.quasar-khhagj.root.comops.site
21:43:57.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4674730
21:43:57.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:43:57.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:57.707504 1532 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:43:57.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:57.709069 1532 file_cache.cc:493] Constructed file cache file cache with capacity 419430
21:43:57.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:57.728704 1541 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:43:58.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:57.728691 1538 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:43:58.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:57.729094 1539 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:43:58.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:57.730705 1532 server_base.cc:1034] running on GCE node
21:43:58.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:58.904619 1532 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:43:58.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:58.907845 1532 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:43:58.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:58.909327 1532 hybrid_clock.cc:648] HybridClock initialized: now 1790459038909263 us; error 73 us; skew 500 ppm
21:43:58.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:58.910300 1532 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:43:58.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 21:43:58 dist-test-slave-b0jg krb5kdc[1110](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790459038, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.18.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:43:58.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:58.917198 1532 init.cc:377] Logged in from keytab as kudu/127.1.18.188@KRBTEST.COM (short username kudu)
21:43:58.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:58.923305 1532 webserver.cc:460] Webserver started at http://127.1.18.188:40443/ using document root <none> and password file <none>
21:43:58.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:58.924407 1532 fs_manager.cc:359] Metadata directory not provided
21:43:58.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:58.924747 1532 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:43:58.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:58.936077 1532 fs_manager.cc:709] Time spent opening directory manager: real 0.007s user 0.007s sys 0.001s
21:43:58.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:58.941658 1551 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:43:58.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:58.943074 1532 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.005s sys 0.001s
21:43:58.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:58.943459 1532 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-2/data,/tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-2/wal
21:43:58.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2b5583981b4048d2929b1e83236e47b2"
21:43:58.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 21:43:52 on dist-test-slave-b0jg"
21:43:58.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:58.945964 1532 fs_report.cc:383] FS layout report
21:43:58.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:43:58.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-2/wal
21:43:58.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-2/wal
21:43:58.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-2/data/data
21:43:58.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:43:58.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:43:58.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:43:58.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:43:58.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:43:58.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:43:58.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:43:58.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:43:58.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:43:58.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:43:58.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:43:58.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:58.993836 1532 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
21:43:59.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:59.000443 1532 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:43:59.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:59.001053 1532 kserver.cc:163] Server-wide thread pool size limit: 3276
21:43:59.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:59.089654 1532 rpc_server.cc:307] RPC server started. Bound to: 127.1.18.188:35079
21:43:59.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:59.089715 1602 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.18.188:35079 every 8 connection(s)
21:43:59.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:59.092816 1532 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskGzKGwb/test-tmp/mini-kudu-cluster4226861945099900624/master-2/data/info.pb
21:43:59.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:59.096428 1098 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskGzKGwb/build/tsan/bin/kudu as pid 1532
21:43:59.097 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:43:59.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:59.106359 1603 sys_catalog.cc:263] Verifying existing consensus state
21:43:59.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:59.113296 1603 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2: Bootstrap starting.
21:43:59.154 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.18.190:44871(127.1.18.190:44871)] connecting to peer
21:43:59.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:59.166417 1603 log.cc:824] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2: Log is configured to *not* fsync() on all Append() calls
21:43:59.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:59.175503 1603 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:43:59.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:59.176558 1603 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2: Bootstrap complete.
21:43:59.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:59.215790 1603 raft_consensus.cc:348] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bf123b719a624f9b99cc92ce3a1533b8" member_type: VOTER last_known_addr { host: "127.1.18.190" port: 44871 } } peers { permanent_uuid: "d22a615f9b324248ab926b797decf072" member_type: VOTER last_known_addr { host: "127.1.18.189" port: 41451 } } peers { permanent_uuid: "2b5583981b4048d2929b1e83236e47b2" member_type: VOTER last_known_addr { host: "127.1.18.188" port: 35079 } }
21:43:59.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:59.219496 1603 raft_consensus.cc:374] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:43:59.219 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.18.189:41451(127.1.18.189:41451)] connecting to peer
21:43:59.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:59.219842 1603 raft_consensus.cc:729] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2b5583981b4048d2929b1e83236e47b2, State: Initialized, Role: FOLLOWER
21:43:59.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:59.222258 1603 consensus_queue.cc:260] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bf123b719a624f9b99cc92ce3a1533b8" member_type: VOTER last_known_addr { host: "127.1.18.190" port: 44871 } } peers { permanent_uuid: "d22a615f9b324248ab926b797decf072" member_type: VOTER last_known_addr { host: "127.1.18.189" port: 41451 } } peers { permanent_uuid: "2b5583981b4048d2929b1e83236e47b2" member_type: VOTER last_known_addr { host: "127.1.18.188" port: 35079 } }
21:43:59.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:59.226192 1609 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bf123b719a624f9b99cc92ce3a1533b8" member_type: VOTER last_known_addr { host: "127.1.18.190" port: 44871 } } peers { permanent_uuid: "d22a615f9b324248ab926b797decf072" member_type: VOTER last_known_addr { host: "127.1.18.189" port: 41451 } } peers { permanent_uuid: "2b5583981b4048d2929b1e83236e47b2" member_type: VOTER last_known_addr { host: "127.1.18.188" port: 35079 } } }
21:43:59.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:59.227442 1609 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2 [sys.catalog]: This master's current role is: FOLLOWER
21:43:59.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:59.228541 1603 sys_catalog.cc:564] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2 [sys.catalog]: configured and running, proceeding with master startup.
21:43:59.231 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.18.188:35079(127.1.18.188:35079)] connecting to peer
21:43:59.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:59.256968 1557 tablet.cc:2369] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:43:59.293 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:43:59.294 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.189:41451(127.1.18.189:41451)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:43:59.296 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:43:59.299 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.190:44871(127.1.18.190:44871)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:43:59.301 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.190:44871: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:43:59.303 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.1.18.188:35079(127.1.18.188:35079)] Successfully connected to peer
21:43:59.305 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.189:41451: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:43:59.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:59.343582 1622 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:43:59.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:59.344010 1622 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:43:59.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:59.338948 1532 master_runner.cc:186] Error getting master registration for 127.1.18.190:44871: Network error: Client connection negotiation failed: client connection to 127.1.18.190:44871: connect: Connection refused (error 111),
21:43:59.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:59.372664 1532 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:43:59.421 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
21:43:59.434 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.1.18.188
21:43:59.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:59.663794 1609 raft_consensus.cc:482] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:43:59.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:59.664423 1609 raft_consensus.cc:504] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bf123b719a624f9b99cc92ce3a1533b8" member_type: VOTER last_known_addr { host: "127.1.18.190" port: 44871 } } peers { permanent_uuid: "d22a615f9b324248ab926b797decf072" member_type: VOTER last_known_addr { host: "127.1.18.189" port: 41451 } } peers { permanent_uuid: "2b5583981b4048d2929b1e83236e47b2" member_type: VOTER last_known_addr { host: "127.1.18.188" port: 35079 } }
21:43:59.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:59.669894 1609 leader_election.cc:288] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers bf123b719a624f9b99cc92ce3a1533b8 (127.1.18.190:44871), d22a615f9b324248ab926b797decf072 (127.1.18.189:41451)
21:43:59.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:59.671149 1552 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.18.190:44871: connect: Connection refused (error 111)
21:43:59.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:59.684900 1554 leader_election.cc:334] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d22a615f9b324248ab926b797decf072 (127.1.18.189:41451): Network error: Client connection negotiation failed: client connection to 127.1.18.189:41451: connect: Connection refused (error 111)
21:43:59.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:43:59.688320 1552 leader_election.cc:334] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer bf123b719a624f9b99cc92ce3a1533b8 (127.1.18.190:44871): Network error: Client connection negotiation failed: client connection to 127.1.18.190:44871: connect: Connection refused (error 111)
21:43:59.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:59.688807 1552 leader_election.cc:302] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 2b5583981b4048d2929b1e83236e47b2; no voters: bf123b719a624f9b99cc92ce3a1533b8, d22a615f9b324248ab926b797decf072
21:43:59.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 21:43:59 dist-test-slave-b0jg krb5kdc[1110](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790459033, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.18.188@KRBTEST.COM
21:43:59.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:43:59.689915 1609 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
21:43:59.739 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x93d58803, L:/127.0.0.1:33996 - R:/127.1.18.188:35079] using SASL/GSSAPI
21:43:59.763 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079; will retry
21:43:59.784 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.190:44871(127.1.18.190:44871)] connecting to peer
21:43:59.789 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.189:41451(127.1.18.189:41451)] connecting to peer
21:43:59.789 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:43:59.790 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.190:44871(127.1.18.190:44871)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:43:59.791 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.190:44871: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:43:59.796 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:43:59.797 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.189:41451(127.1.18.189:41451)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:43:59.798 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.189:41451: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:43:59.801 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079; will retry
21:43:59.823 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.190:44871(127.1.18.190:44871)] connecting to peer
21:43:59.826 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.189:41451(127.1.18.189:41451)] connecting to peer
21:43:59.826 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:43:59.827 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.190:44871(127.1.18.190:44871)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:43:59.828 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.190:44871: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:43:59.833 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:43:59.833 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.189:41451(127.1.18.189:41451)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:43:59.834 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.189:41451: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:43:59.836 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079; will retry
21:43:59.843 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.190:44871(127.1.18.190:44871)] connecting to peer
21:43:59.844 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.189:41451(127.1.18.189:41451)] connecting to peer
21:43:59.846 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:43:59.846 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.190:44871(127.1.18.190:44871)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:43:59.846 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.190:44871: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:43:59.851 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:43:59.851 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.189:41451(127.1.18.189:41451)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:43:59.852 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.189:41451: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:43:59.854 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079; will retry
21:43:59.883 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.190:44871(127.1.18.190:44871)] connecting to peer
21:43:59.885 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.189:41451(127.1.18.189:41451)] connecting to peer
21:43:59.885 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:43:59.886 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.190:44871(127.1.18.190:44871)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:43:59.887 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.190:44871: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:43:59.890 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:43:59.890 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.189:41451(127.1.18.189:41451)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:43:59.891 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.189:41451: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:43:59.893 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079; will retry
21:43:59.903 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.190:44871(127.1.18.190:44871)] connecting to peer
21:43:59.904 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.189:41451(127.1.18.189:41451)] connecting to peer
21:43:59.905 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:43:59.905 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.190:44871(127.1.18.190:44871)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:43:59.906 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.190:44871: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:43:59.908 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:43:59.908 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.189:41451(127.1.18.189:41451)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:43:59.909 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.189:41451: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:43:59.912 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079; will retry
21:43:59.942 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.190:44871(127.1.18.190:44871)] connecting to peer
21:43:59.945 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:43:59.945 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.190:44871(127.1.18.190:44871)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:43:59.946 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.189:41451(127.1.18.189:41451)] connecting to peer
21:43:59.946 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.190:44871: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:43:59.951 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:43:59.952 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.189:41451(127.1.18.189:41451)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:43:59.953 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.189:41451: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:43:59.955 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079; will retry
21:44:00.043 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.190:44871(127.1.18.190:44871)] connecting to peer
21:44:00.045 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.189:41451(127.1.18.189:41451)] connecting to peer
21:44:00.046 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:44:00.046 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.190:44871(127.1.18.190:44871)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:44:00.047 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.190:44871: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:44:00.050 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:44:00.050 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.189:41451(127.1.18.189:41451)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:44:00.050 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.189:41451: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:44:00.064 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079; will retry
21:44:00.222 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.190:44871(127.1.18.190:44871)] connecting to peer
21:44:00.224 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.189:41451(127.1.18.189:41451)] connecting to peer
21:44:00.225 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:44:00.225 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.190:44871(127.1.18.190:44871)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:44:00.228 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.190:44871: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:44:00.230 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:44:00.230 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.189:41451(127.1.18.189:41451)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:44:00.230 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.189:41451: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:44:00.233 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079; will retry
21:44:00.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:00.347471 1622 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:44:00.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:00.348093 1622 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:44:00.622 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.190:44871(127.1.18.190:44871)] connecting to peer
21:44:00.624 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.189:41451(127.1.18.189:41451)] connecting to peer
21:44:00.625 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:44:00.625 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.190:44871(127.1.18.190:44871)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:44:00.627 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.190:44871: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:44:00.628 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:44:00.629 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.189:41451(127.1.18.189:41451)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:44:00.631 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.189:41451: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:44:00.632 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079; will retry
21:44:01.003 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.190:44871(127.1.18.190:44871)] connecting to peer
21:44:01.004 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.189:41451(127.1.18.189:41451)] connecting to peer
21:44:01.005 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:44:01.005 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.190:44871(127.1.18.190:44871)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:44:01.005 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.190:44871: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:44:01.009 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:44:01.009 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.189:41451(127.1.18.189:41451)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:44:01.010 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.189:41451: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:44:01.014 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079; will retry
21:44:01.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:01.351594 1622 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:44:01.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:01.351912 1622 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:44:01.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:44:01.843448 1630 raft_consensus.cc:482] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:44:01.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:44:01.843950 1630 raft_consensus.cc:504] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bf123b719a624f9b99cc92ce3a1533b8" member_type: VOTER last_known_addr { host: "127.1.18.190" port: 44871 } } peers { permanent_uuid: "d22a615f9b324248ab926b797decf072" member_type: VOTER last_known_addr { host: "127.1.18.189" port: 41451 } } peers { permanent_uuid: "2b5583981b4048d2929b1e83236e47b2" member_type: VOTER last_known_addr { host: "127.1.18.188" port: 35079 } }
21:44:01.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:44:01.845691 1630 leader_election.cc:288] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers bf123b719a624f9b99cc92ce3a1533b8 (127.1.18.190:44871), d22a615f9b324248ab926b797decf072 (127.1.18.189:41451)
21:44:01.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:01.852527 1554 leader_election.cc:334] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d22a615f9b324248ab926b797decf072 (127.1.18.189:41451): Network error: Client connection negotiation failed: client connection to 127.1.18.189:41451: connect: Connection refused (error 111)
21:44:01.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:01.852941 1552 leader_election.cc:334] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer bf123b719a624f9b99cc92ce3a1533b8 (127.1.18.190:44871): Network error: Client connection negotiation failed: client connection to 127.1.18.190:44871: connect: Connection refused (error 111)
21:44:01.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:44:01.853309 1552 leader_election.cc:302] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 2b5583981b4048d2929b1e83236e47b2; no voters: bf123b719a624f9b99cc92ce3a1533b8, d22a615f9b324248ab926b797decf072
21:44:01.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:44:01.854043 1630 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
21:44:02.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:02.355286 1622 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:44:02.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:02.355595 1622 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:44:02.903 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.190:44871(127.1.18.190:44871)] connecting to peer
21:44:02.905 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.189:41451(127.1.18.189:41451)] connecting to peer
21:44:02.905 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:44:02.906 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.190:44871(127.1.18.190:44871)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:44:02.907 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.190:44871: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:44:02.908 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:44:02.909 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.189:41451(127.1.18.189:41451)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:44:02.909 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.189:41451: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:44:02.912 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079; will retry
21:44:03.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:03.358784 1622 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:44:03.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:03.359166 1622 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:44:03.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:44:03.534919 1636 raft_consensus.cc:482] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:44:03.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:44:03.535429 1636 raft_consensus.cc:504] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bf123b719a624f9b99cc92ce3a1533b8" member_type: VOTER last_known_addr { host: "127.1.18.190" port: 44871 } } peers { permanent_uuid: "d22a615f9b324248ab926b797decf072" member_type: VOTER last_known_addr { host: "127.1.18.189" port: 41451 } } peers { permanent_uuid: "2b5583981b4048d2929b1e83236e47b2" member_type: VOTER last_known_addr { host: "127.1.18.188" port: 35079 } }
21:44:03.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:44:03.537012 1636 leader_election.cc:288] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers bf123b719a624f9b99cc92ce3a1533b8 (127.1.18.190:44871), d22a615f9b324248ab926b797decf072 (127.1.18.189:41451)
21:44:03.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:03.542521 1552 leader_election.cc:334] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer bf123b719a624f9b99cc92ce3a1533b8 (127.1.18.190:44871): Network error: Client connection negotiation failed: client connection to 127.1.18.190:44871: connect: Connection refused (error 111)
21:44:03.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:03.543038 1554 leader_election.cc:334] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d22a615f9b324248ab926b797decf072 (127.1.18.189:41451): Network error: Client connection negotiation failed: client connection to 127.1.18.189:41451: connect: Connection refused (error 111)
21:44:03.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:44:03.543426 1554 leader_election.cc:302] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 2b5583981b4048d2929b1e83236e47b2; no voters: bf123b719a624f9b99cc92ce3a1533b8, d22a615f9b324248ab926b797decf072
21:44:03.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:44:03.543998 1636 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
21:44:04.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:04.362394 1622 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:44:04.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:04.362692 1622 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:44:04.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:44:04.375644 1532 master_runner.cc:186] Error getting master registration for 127.1.18.190:44871: Network error: Client connection negotiation failed: client connection to 127.1.18.190:44871: connect: Connection refused (error 111),
21:44:04.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:44:04.375998 1532 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:44:04.842 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.190:44871(127.1.18.190:44871)] connecting to peer
21:44:04.843 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.189:41451(127.1.18.189:41451)] connecting to peer
21:44:04.845 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:44:04.845 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.190:44871(127.1.18.190:44871)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:44:04.846 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.190:44871: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:44:04.848 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:44:04.848 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.189:41451(127.1.18.189:41451)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:44:04.849 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.189:41451: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:44:04.851 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079; will retry
21:44:05.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:05.365976 1622 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:44:05.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:05.366285 1622 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:44:05.582 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.190:44871(127.1.18.190:44871)] connecting to peer
21:44:05.583 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.189:41451(127.1.18.189:41451)] connecting to peer
21:44:05.585 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:44:05.585 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.190:44871(127.1.18.190:44871)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:44:05.586 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.190:44871: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:44:05.586 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:44:05.587 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.189:41451(127.1.18.189:41451)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:44:05.587 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.189:41451: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:44:05.589 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079; will retry
21:44:05.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:44:05.776072 1643 raft_consensus.cc:482] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:44:05.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:44:05.776453 1643 raft_consensus.cc:504] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bf123b719a624f9b99cc92ce3a1533b8" member_type: VOTER last_known_addr { host: "127.1.18.190" port: 44871 } } peers { permanent_uuid: "d22a615f9b324248ab926b797decf072" member_type: VOTER last_known_addr { host: "127.1.18.189" port: 41451 } } peers { permanent_uuid: "2b5583981b4048d2929b1e83236e47b2" member_type: VOTER last_known_addr { host: "127.1.18.188" port: 35079 } }
21:44:05.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:44:05.777911 1643 leader_election.cc:288] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers bf123b719a624f9b99cc92ce3a1533b8 (127.1.18.190:44871), d22a615f9b324248ab926b797decf072 (127.1.18.189:41451)
21:44:05.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:05.780087 1552 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.18.190:44871: connect: Connection refused (error 111) [suppressed 5 similar messages]
21:44:05.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:05.783066 1554 leader_election.cc:334] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d22a615f9b324248ab926b797decf072 (127.1.18.189:41451): Network error: Client connection negotiation failed: client connection to 127.1.18.189:41451: connect: Connection refused (error 111)
21:44:05.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:05.783638 1552 leader_election.cc:334] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer bf123b719a624f9b99cc92ce3a1533b8 (127.1.18.190:44871): Network error: Client connection negotiation failed: client connection to 127.1.18.190:44871: connect: Connection refused (error 111)
21:44:05.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:44:05.784111 1552 leader_election.cc:302] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 2b5583981b4048d2929b1e83236e47b2; no voters: bf123b719a624f9b99cc92ce3a1533b8, d22a615f9b324248ab926b797decf072
21:44:05.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:44:05.784919 1643 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
21:44:06.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:06.370011 1622 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:44:06.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:06.370335 1622 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:44:07.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:07.373289 1622 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:44:07.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:07.373564 1622 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:44:08.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:08.377099 1622 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:44:08.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:08.377430 1622 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:44:08.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 21:44:08 dist-test-slave-b0jg krb5kdc[1110](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790459048, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.18.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:44:08.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:44:08.921876 1547 init.cc:295] Successfully reacquired a new kerberos TGT
21:44:09.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:44:09.380589 1532 master_runner.cc:186] Error getting master registration for 127.1.18.190:44871: Network error: Client connection negotiation failed: client connection to 127.1.18.190:44871: connect: Connection refused (error 111),
21:44:09.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:09.380980 1622 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:44:09.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:44:09.381170 1532 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:44:09.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:09.381239 1622 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:44:09.542 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.190:44871(127.1.18.190:44871)] connecting to peer
21:44:09.544 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.189:41451(127.1.18.189:41451)] connecting to peer
21:44:09.545 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:44:09.546 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.190:44871(127.1.18.190:44871)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:44:09.546 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.190:44871: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:44:09.547 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:44:09.548 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.189:41451(127.1.18.189:41451)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:44:09.548 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.189:41451: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:44:09.551 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079; will retry
21:44:10.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:10.384153 1622 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:44:10.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:10.384433 1622 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:44:10.443 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.190:44871(127.1.18.190:44871)] connecting to peer
21:44:10.444 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.189:41451(127.1.18.189:41451)] connecting to peer
21:44:10.445 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:44:10.446 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.190:44871(127.1.18.190:44871)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:44:10.447 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.190:44871: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:44:10.448 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:44:10.448 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.189:41451(127.1.18.189:41451)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:44:10.448 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.189:41451: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:44:10.451 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079; will retry
21:44:11.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:11.396246 1622 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:44:11.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:11.397797 1622 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:44:11.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:44:11.519469 1653 raft_consensus.cc:482] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:44:11.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:44:11.520131 1653 raft_consensus.cc:504] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bf123b719a624f9b99cc92ce3a1533b8" member_type: VOTER last_known_addr { host: "127.1.18.190" port: 44871 } } peers { permanent_uuid: "d22a615f9b324248ab926b797decf072" member_type: VOTER last_known_addr { host: "127.1.18.189" port: 41451 } } peers { permanent_uuid: "2b5583981b4048d2929b1e83236e47b2" member_type: VOTER last_known_addr { host: "127.1.18.188" port: 35079 } }
21:44:11.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:44:11.522336 1653 leader_election.cc:288] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers bf123b719a624f9b99cc92ce3a1533b8 (127.1.18.190:44871), d22a615f9b324248ab926b797decf072 (127.1.18.189:41451)
21:44:11.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:11.524849 1552 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.18.190:44871: connect: Connection refused (error 111) [suppressed 1 similar messages]
21:44:11.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:11.528262 1554 leader_election.cc:334] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d22a615f9b324248ab926b797decf072 (127.1.18.189:41451): Network error: Client connection negotiation failed: client connection to 127.1.18.189:41451: connect: Connection refused (error 111)
21:44:11.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:11.528766 1552 leader_election.cc:334] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer bf123b719a624f9b99cc92ce3a1533b8 (127.1.18.190:44871): Network error: Client connection negotiation failed: client connection to 127.1.18.190:44871: connect: Connection refused (error 111)
21:44:11.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:44:11.529125 1552 leader_election.cc:302] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 2b5583981b4048d2929b1e83236e47b2; no voters: bf123b719a624f9b99cc92ce3a1533b8, d22a615f9b324248ab926b797decf072
21:44:11.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:44:11.529783 1653 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
21:44:12.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:12.399729 801 debug-util.cc:398] Leaking SignalData structure 0x7b08000afc80 after lost signal to thread 676
21:44:12.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:12.401108 801 debug-util.cc:398] Leaking SignalData structure 0x7b08000bcb20 after lost signal to thread 804
21:44:12.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:12.401806 1622 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:44:12.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:12.402101 1622 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:44:12.642 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.190:44871(127.1.18.190:44871)] connecting to peer
21:44:12.644 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.189:41451(127.1.18.189:41451)] connecting to peer
21:44:12.645 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:44:12.646 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.190:44871(127.1.18.190:44871)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:44:12.646 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.190:44871: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:44:12.647 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:44:12.647 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.189:41451(127.1.18.189:41451)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:44:12.647 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.189:41451: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:44:12.650 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079; will retry
21:44:12.722 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.190:44871(127.1.18.190:44871)] connecting to peer
21:44:12.723 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.189:41451(127.1.18.189:41451)] connecting to peer
21:44:12.725 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:44:12.726 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.190:44871(127.1.18.190:44871)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:44:12.726 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.190:44871: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:44:12.728 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:44:12.729 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.189:41451(127.1.18.189:41451)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:44:12.729 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.189:41451: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:44:12.731 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079; will retry
21:44:12.983 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.190:44871(127.1.18.190:44871)] connecting to peer
21:44:12.985 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.189:41451(127.1.18.189:41451)] connecting to peer
21:44:12.985 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:44:12.985 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.190:44871(127.1.18.190:44871)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:44:12.986 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.190:44871: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:44:12.988 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:44:12.988 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.189:41451(127.1.18.189:41451)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:44:12.989 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.189:41451: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:44:12.991 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079; will retry
21:44:13.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:13.404929 1622 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:44:13.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:13.405207 1622 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:44:14.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:44:14.244182 1659 raft_consensus.cc:482] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:44:14.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:44:14.244732 1659 raft_consensus.cc:504] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bf123b719a624f9b99cc92ce3a1533b8" member_type: VOTER last_known_addr { host: "127.1.18.190" port: 44871 } } peers { permanent_uuid: "d22a615f9b324248ab926b797decf072" member_type: VOTER last_known_addr { host: "127.1.18.189" port: 41451 } } peers { permanent_uuid: "2b5583981b4048d2929b1e83236e47b2" member_type: VOTER last_known_addr { host: "127.1.18.188" port: 35079 } }
21:44:14.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:44:14.246918 1659 leader_election.cc:288] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers bf123b719a624f9b99cc92ce3a1533b8 (127.1.18.190:44871), d22a615f9b324248ab926b797decf072 (127.1.18.189:41451)
21:44:14.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:14.252861 1552 leader_election.cc:334] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer bf123b719a624f9b99cc92ce3a1533b8 (127.1.18.190:44871): Network error: Client connection negotiation failed: client connection to 127.1.18.190:44871: connect: Connection refused (error 111)
21:44:14.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:14.253315 1554 leader_election.cc:334] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d22a615f9b324248ab926b797decf072 (127.1.18.189:41451): Network error: Client connection negotiation failed: client connection to 127.1.18.189:41451: connect: Connection refused (error 111)
21:44:14.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:44:14.253839 1554 leader_election.cc:302] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 2b5583981b4048d2929b1e83236e47b2; no voters: bf123b719a624f9b99cc92ce3a1533b8, d22a615f9b324248ab926b797decf072
21:44:14.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:44:14.254474 1659 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
21:44:14.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:44:14.383325 1532 master_runner.cc:186] Error getting master registration for 127.1.18.190:44871: Network error: Client connection negotiation failed: client connection to 127.1.18.190:44871: connect: Connection refused (error 111),
21:44:14.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:44:14.383785 1532 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:44:14.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:14.408149 1622 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:44:14.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:14.408442 1622 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:44:15.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:15.411510 1622 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:44:15.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:15.411777 1622 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:44:16.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:16.414600 1622 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:44:16.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:16.414903 1622 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:44:16.422 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.190:44871(127.1.18.190:44871)] connecting to peer
21:44:16.424 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.189:41451(127.1.18.189:41451)] connecting to peer
21:44:16.425 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:44:16.426 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.190:44871(127.1.18.190:44871)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:44:16.426 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:44:16.426 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.189:41451(127.1.18.189:41451)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:44:16.427 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.190:44871: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:44:16.427 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.189:41451: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:44:16.431 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079; will retry
21:44:17.162 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.190:44871(127.1.18.190:44871)] connecting to peer
21:44:17.163 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.189:41451(127.1.18.189:41451)] connecting to peer
21:44:17.164 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:44:17.164 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.190:44871(127.1.18.190:44871)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:44:17.164 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.190:44871: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:44:17.166 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:44:17.166 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.189:41451(127.1.18.189:41451)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:44:17.166 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.189:41451: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:44:17.168 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079; will retry
21:44:17.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:17.417891 1622 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:44:17.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:17.418162 1622 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:44:18.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:18.421803 1622 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:44:18.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:18.422142 1622 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:44:18.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 21:44:18 dist-test-slave-b0jg krb5kdc[1110](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790459058, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.18.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:44:18.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:44:18.924996 1547 init.cc:295] Successfully reacquired a new kerberos TGT
21:44:19.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:44:19.387512 1532 master_runner.cc:186] Error getting master registration for 127.1.18.190:44871: Network error: Client connection negotiation failed: client connection to 127.1.18.190:44871: connect: Connection refused (error 111),
21:44:19.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:44:19.387921 1532 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:44:19.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:19.425346 1622 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:44:19.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:19.425629 1622 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:44:19.622 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.190:44871(127.1.18.190:44871)] connecting to peer
21:44:19.623 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.189:41451(127.1.18.189:41451)] connecting to peer
21:44:19.624 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:44:19.624 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.190:44871(127.1.18.190:44871)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:44:19.624 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.190:44871: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:44:19.625 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:44:19.625 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.189:41451(127.1.18.189:41451)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:44:19.626 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.189:41451: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:44:19.629 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079; will retry
21:44:19.883 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.190:44871(127.1.18.190:44871)] connecting to peer
21:44:19.884 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.189:41451(127.1.18.189:41451)] connecting to peer
21:44:19.885 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:44:19.885 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.190:44871(127.1.18.190:44871)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:44:19.885 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.190:44871: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:44:19.886 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:44:19.887 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.189:41451(127.1.18.189:41451)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:44:19.888 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.189:41451: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:44:19.890 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079; will retry
21:44:20.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:20.428699 1622 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:44:20.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:20.429008 1622 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:44:21.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:21.432312 1622 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:44:21.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:21.432643 1622 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:44:22.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:22.435768 1622 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:44:22.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:22.436077 1622 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:44:22.742 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.190:44871(127.1.18.190:44871)] connecting to peer
21:44:22.743 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.189:41451(127.1.18.189:41451)] connecting to peer
21:44:22.744 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:44:22.744 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.190:44871(127.1.18.190:44871)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:44:22.744 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.190:44871: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:44:22.745 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:44:22.745 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.189:41451(127.1.18.189:41451)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:44:22.745 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.189:41451: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:44:22.748 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079; will retry
21:44:23.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:23.439214 1622 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:44:23.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:23.439509 1622 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:44:23.962 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.190:44871(127.1.18.190:44871)] connecting to peer
21:44:23.963 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.189:41451(127.1.18.189:41451)] connecting to peer
21:44:23.964 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:44:23.964 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.190:44871(127.1.18.190:44871)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:44:23.964 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.190:44871: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:44:23.965 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:44:23.965 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.189:41451(127.1.18.189:41451)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:44:23.965 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.189:41451: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:44:23.968 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079; will retry
21:44:24.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:44:24.391098 1532 master_runner.cc:186] Error getting master registration for 127.1.18.190:44871: Network error: Client connection negotiation failed: client connection to 127.1.18.190:44871: connect: Connection refused (error 111),
21:44:24.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:44:24.391577 1532 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:44:24.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:24.442791 1622 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:44:24.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:24.443205 1622 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:44:25.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:25.446579 1622 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:44:25.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:25.446904 1622 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:44:26.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:26.450229 1622 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:44:26.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:26.450512 1622 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:44:27.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:27.453553 1622 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2b5583981b4048d2929b1e83236e47b2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:44:27.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 21:44:27.453821 1622 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:44:27.583 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.190:44871(127.1.18.190:44871)] connecting to peer
21:44:27.583 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.189:41451(127.1.18.189:41451)] connecting to peer
21:44:27.584 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:44:27.584 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.190:44871(127.1.18.190:44871)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:44:27.584 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.190:44871: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871
21:44:27.586 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:44:27.586 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.189:41451(127.1.18.189:41451)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:44:27.586 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.189:41451: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
21:44:27.588 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079; will retry
21:44:27.594 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=28488), Traces: [0ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:44871, [112ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:41451, [124ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:35079, [193ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:44871: Network error: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871, [188ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:41451: Network error: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [650ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:35079: OK, [657ms] delaying RPC due to: Service unavailable: Master config (127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [676ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:44871, [682ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:41451, [683ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:44871: Network error: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871, [684ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:35079, [690ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:41451: Network error: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [693ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:35079: OK, [698ms] delaying RPC due to: Service unavailable: Master config (127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [715ms] refreshing cache from master, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:44871, [719ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:41451, [720ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:44871: Network error: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871, [721ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:35079, [726ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:41451: Network error: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [728ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:35079: OK, [730ms] delaying RPC due to: Service unavailable: Master config (127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [735ms] refreshing cache from master, [735ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:44871, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:41451, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:35079, [739ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:44871: Network error: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871, [744ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:41451: Network error: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [746ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:35079: OK, [747ms] delaying RPC due to: Service unavailable: Master config (127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [775ms] refreshing cache from master, [775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:44871, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:41451, [779ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:44871: Network error: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871, [779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:35079, [784ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:41451: Network error: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [786ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:35079: OK, [787ms] delaying RPC due to: Service unavailable: Master config (127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [795ms] refreshing cache from master, [795ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:44871, [797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:41451, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:35079, [798ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:44871: Network error: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871, [801ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:41451: Network error: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [804ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:35079: OK, [805ms] delaying RPC due to: Service unavailable: Master config (127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [834ms] refreshing cache from master, [835ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:44871, [837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:41451, [838ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:44871: Network error: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871, [840ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:35079, [845ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:41451: Network error: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [848ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:35079: OK, [849ms] delaying RPC due to: Service unavailable: Master config (127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [935ms] refreshing cache from master, [935ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:44871, [938ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:41451, [939ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:35079, [939ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:44871: Network error: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871, [943ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:41451: Network error: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [955ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:35079: OK, [958ms] delaying RPC due to: Service unavailable: Master config (127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [1114ms] refreshing cache from master, [1115ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:44871, [1116ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:41451, [1118ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:35079, [1118ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:44871: Network error: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871, [1123ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:41451: Network error: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [1125ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:35079: OK, [1127ms] delaying RPC due to: Service unavailable: Master config (127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [1514ms] refreshing cache from master, [1515ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:44871, [1517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:41451, [1519ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:35079, [1519ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:44871: Network error: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871, [1522ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:41451: Network error: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [1525ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:35079: OK, [1526ms] delaying RPC due to: Service unavailable: Master config (127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [1895ms] refreshing cache from master, [1895ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:44871, [1897ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:41451, [1898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:35079, [1898ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:44871: Network error: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871, [1902ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:41451: Network error: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [1907ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:35079: OK, [1907ms] delaying RPC due to: Service unavailable: Master config (127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [3795ms] refreshing cache from master, [3796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:44871, [3797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:41451, [3799ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:35079, [3799ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:44871: Network error: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871, [3802ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:41451: Network error: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [3804ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:35079: OK, [3805ms] delaying RPC due to: Service unavailable: Master config (127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [5734ms] refreshing cache from master, [5735ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:44871, [5736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:41451, [5737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:35079, [5738ms] too many traces: truncated at 100 traces, deferred=Deferred@818840767(state=PENDING, result=null, callback=(continuation of Deferred@109207934 after retry RPC after error@38856450) -> (continuation of Deferred@1504326812 after retry RPC after error@486910058) -> (continuation of Deferred@135901989 after retry RPC after error@225721811) -> (continuation of Deferred@91420213 after retry RPC after error@889477129) -> (continuation of Deferred@1891718423 after retry RPC after error@1551332473) -> (continuation of Deferred@1319563932 after retry RPC after error@1509244753) -> (continuation of Deferred@593932437 after retry RPC after error@1154862140) -> (continuation of Deferred@1025438760 after retry RPC after error@1306610541) -> (continuation of Deferred@1536534620 after retry RPC after error@827013950) -> (continuation of Deferred@2067674763 after retry RPC after error@1011069090) -> (continuation of Deferred@1247946814 after retry RPC after error@580413575) -> (continuation of Deferred@672700465 after retry RPC after error@167766832) -> (continuation of Deferred@1376190953 after retry RPC after error@867862398) -> (continuation of Deferred@1594837032 after retry RPC after error@702516882) -> (continuation of Deferred@1136511497 after retry RPC after error@618345591) -> (continuation of Deferred@28337868 after retry RPC after error@1422836598) -> (continuation of Deferred@2039391844 after retry RPC after error@76238254) -> (continuation of Deferred@101703157 after retry RPC after error@620260145) -> (continuation of Deferred@684278860 after retry RPC after error@1995545286) -> (continuation of Deferred@250155416 after retry RPC after error@1764709015) -> (continuation of Deferred@195465434 after retry RPC after error@1912282306) -> (continuation of Deferred@936945332 after retry RPC after error@611381554) -> (continuation of Deferred@460297510 after retry RPC after error@853247877) -> (continuation of Deferred@1086618220 after retry RPC after error@1671359015) -> (continuation of Deferred@71275991 after retry RPC after error@1790365050), errback=(continuation of Deferred@109207934 after retry RPC after error@38856450) -> (continuation of Deferred@1504326812 after retry RPC after error@486910058) -> (continuation of Deferred@135901989 after retry RPC after error@225721811) -> (continuation of Deferred@91420213 after retry RPC after error@889477129) -> (continuation of Deferred@1891718423 after retry RPC after error@1551332473) -> (continuation of Deferred@1319563932 after retry RPC after error@1509244753) -> (continuation of Deferred@593932437 after retry RPC after error@1154862140) -> (continuation of Deferred@1025438760 after retry RPC after error@1306610541) -> (continuation of Deferred@1536534620 after retry RPC after error@827013950) -> (continuation of Deferred@2067674763 after retry RPC after error@1011069090) -> (continuation of Deferred@1247946814 after retry RPC after error@580413575) -> (continuation of Deferred@672700465 after retry RPC after error@167766832) -> (continuation of Deferred@1376190953 after retry RPC after error@867862398) -> (continuation of Deferred@1594837032 after retry RPC after error@702516882) -> (continuation of Deferred@1136511497 after retry RPC after error@618345591) -> (continuation of Deferred@28337868 after retry RPC after error@1422836598) -> (continuation of Deferred@2039391844 after retry RPC after error@76238254) -> (continuation of Deferred@101703157 after retry RPC after error@620260145) -> (continuation of Deferred@684278860 after retry RPC after error@1995545286) -> (continuation of Deferred@250155416 after retry RPC after error@1764709015) -> (continuation of Deferred@195465434 after retry RPC after error@1912282306) -> (continuation of Deferred@936945332 after retry RPC after error@611381554) -> (continuation of Deferred@460297510 after retry RPC after error@853247877) -> (continuation of Deferred@1086618220 after retry RPC after error@1671359015) -> (continuation of Deferred@71275991 after retry RPC after error@1790365050)))
21:44:27.599 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.1.18.188:35079(127.1.18.188:35079)] disconnecting while in state READY
21:44:27.600 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.1.18.190:44871(127.1.18.190:44871)] disconnecting while in state TERMINATED
21:44:27.600 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.1.18.189:41451(127.1.18.189:41451)] disconnecting while in state TERMINATED
21:44:27.607 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
21:44:27.609 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
21:44:27.610 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.1.18.188:35079(127.1.18.188:35079)] handling channelInactive
21:44:27.611 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.188:35079(127.1.18.188:35079)] cleaning up while in state READY due to: connection closed
21:44:27.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:44:27.616366 1098 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskGzKGwb/build/tsan/bin/kudu with pid 1532
21:44:28.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-26T21:44:28Z chronyd exiting
21:44:29.757 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
21:44:29.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:44:29.758813 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskGzKGwb/build/tsan/bin/kudu with pid 675
21:44:29.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:44:29.785934 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskGzKGwb/build/tsan/bin/kudu with pid 815
21:44:29.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:44:29.812709 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskGzKGwb/build/tsan/bin/kudu with pid 953
21:44:29.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:44:29.843174 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskGzKGwb/build/tsan/bin/kudu with pid 386
21:44:29.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:44:29.873608 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskGzKGwb/build/tsan/bin/kudu with pid 456
21:44:29.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 21:44:29.903800 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskGzKGwb/build/tsan/bin/kudu with pid 531
21:44:30.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-26T21:44:30Z chronyd exiting
21:44:32.027 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=28488), Traces: [0ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:44871, [112ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:41451, [124ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:35079, [193ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:44871: Network error: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871, [188ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:41451: Network error: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [650ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:35079: OK, [657ms] delaying RPC due to: Service unavailable: Master config (127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [676ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:44871, [682ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:41451, [683ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:44871: Network error: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871, [684ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:35079, [690ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:41451: Network error: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [693ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:35079: OK, [698ms] delaying RPC due to: Service unavailable: Master config (127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [715ms] refreshing cache from master, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:44871, [719ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:41451, [720ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:44871: Network error: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871, [721ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:35079, [726ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:41451: Network error: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [728ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:35079: OK, [730ms] delaying RPC due to: Service unavailable: Master config (127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [735ms] refreshing cache from master, [735ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:44871, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:41451, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:35079, [739ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:44871: Network error: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871, [744ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:41451: Network error: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [746ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:35079: OK, [747ms] delaying RPC due to: Service unavailable: Master config (127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [775ms] refreshing cache from master, [775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:44871, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:41451, [779ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:44871: Network error: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871, [779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:35079, [784ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:41451: Network error: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [786ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:35079: OK, [787ms] delaying RPC due to: Service unavailable: Master config (127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [795ms] refreshing cache from master, [795ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:44871, [797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:41451, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:35079, [798ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:44871: Network error: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871, [801ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:41451: Network error: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [804ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:35079: OK, [805ms] delaying RPC due to: Service unavailable: Master config (127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [834ms] refreshing cache from master, [835ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:44871, [837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:41451, [838ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:44871: Network error: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871, [840ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:35079, [845ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:41451: Network error: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [848ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:35079: OK, [849ms] delaying RPC due to: Service unavailable: Master config (127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [935ms] refreshing cache from master, [935ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:44871, [938ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:41451, [939ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:35079, [939ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:44871: Network error: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871, [943ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:41451: Network error: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [955ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:35079: OK, [958ms] delaying RPC due to: Service unavailable: Master config (127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [1114ms] refreshing cache from master, [1115ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:44871, [1116ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:41451, [1118ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:35079, [1118ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:44871: Network error: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871, [1123ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:41451: Network error: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [1125ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:35079: OK, [1127ms] delaying RPC due to: Service unavailable: Master config (127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [1514ms] refreshing cache from master, [1515ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:44871, [1517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:41451, [1519ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:35079, [1519ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:44871: Network error: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871, [1522ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:41451: Network error: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [1525ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:35079: OK, [1526ms] delaying RPC due to: Service unavailable: Master config (127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [1895ms] refreshing cache from master, [1895ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:44871, [1897ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:41451, [1898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:35079, [1898ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:44871: Network error: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871, [1902ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:41451: Network error: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [1907ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:35079: OK, [1907ms] delaying RPC due to: Service unavailable: Master config (127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [3795ms] refreshing cache from master, [3796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:44871, [3797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:41451, [3799ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:35079, [3799ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:44871: Network error: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871, [3802ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:41451: Network error: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [3804ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:35079: OK, [3805ms] delaying RPC due to: Service unavailable: Master config (127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [5734ms] refreshing cache from master, [5735ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:44871, [5736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:41451, [5737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:35079, [5738ms] too many traces: truncated at 100 traces, deferred=Deferred@818840767(state=PENDING, result=null, callback=(continuation of Deferred@109207934 after retry RPC after error@38856450) -> (continuation of Deferred@1504326812 after retry RPC after error@486910058) -> (continuation of Deferred@135901989 after retry RPC after error@225721811) -> (continuation of Deferred@91420213 after retry RPC after error@889477129) -> (continuation of Deferred@1891718423 after retry RPC after error@1551332473) -> (continuation of Deferred@1319563932 after retry RPC after error@1509244753) -> (continuation of Deferred@593932437 after retry RPC after error@1154862140) -> (continuation of Deferred@1025438760 after retry RPC after error@1306610541) -> (continuation of Deferred@1536534620 after retry RPC after error@827013950) -> (continuation of Deferred@2067674763 after retry RPC after error@1011069090) -> (continuation of Deferred@1247946814 after retry RPC after error@580413575) -> (continuation of Deferred@672700465 after retry RPC after error@167766832) -> (continuation of Deferred@1376190953 after retry RPC after error@867862398) -> (continuation of Deferred@1594837032 after retry RPC after error@702516882) -> (continuation of Deferred@1136511497 after retry RPC after error@618345591) -> (continuation of Deferred@28337868 after retry RPC after error@1422836598) -> (continuation of Deferred@2039391844 after retry RPC after error@76238254) -> (continuation of Deferred@101703157 after retry RPC after error@620260145) -> (continuation of Deferred@684278860 after retry RPC after error@1995545286) -> (continuation of Deferred@250155416 after retry RPC after error@1764709015) -> (continuation of Deferred@195465434 after retry RPC after error@1912282306) -> (continuation of Deferred@936945332 after retry RPC after error@611381554) -> (continuation of Deferred@460297510 after retry RPC after error@853247877) -> (continuation of Deferred@1086618220 after retry RPC after error@1671359015) -> (continuation of Deferred@71275991 after retry RPC after error@1790365050), errback=(continuation of Deferred@109207934 after retry RPC after error@38856450) -> (continuation of Deferred@1504326812 after retry RPC after error@486910058) -> (continuation of Deferred@135901989 after retry RPC after error@225721811) -> (continuation of Deferred@91420213 after retry RPC after error@889477129) -> (continuation of Deferred@1891718423 after retry RPC after error@1551332473) -> (continuation of Deferred@1319563932 after retry RPC after error@1509244753) -> (continuation of Deferred@593932437 after retry RPC after error@1154862140) -> (continuation of Deferred@1025438760 after retry RPC after error@1306610541) -> (continuation of Deferred@1536534620 after retry RPC after error@827013950) -> (continuation of Deferred@2067674763 after retry RPC after error@1011069090) -> (continuation of Deferred@1247946814 after retry RPC after error@580413575) -> (continuation of Deferred@672700465 after retry RPC after error@167766832) -> (continuation of Deferred@1376190953 after retry RPC after error@867862398) -> (continuation of Deferred@1594837032 after retry RPC after error@702516882) -> (continuation of Deferred@1136511497 after retry RPC after error@618345591) -> (continuation of Deferred@28337868 after retry RPC after error@1422836598) -> (continuation of Deferred@2039391844 after retry RPC after error@76238254) -> (continuation of Deferred@101703157 after retry RPC after error@620260145) -> (continuation of Deferred@684278860 after retry RPC after error@1995545286) -> (continuation of Deferred@250155416 after retry RPC after error@1764709015) -> (continuation of Deferred@195465434 after retry RPC after error@1912282306) -> (continuation of Deferred@936945332 after retry RPC after error@611381554) -> (continuation of Deferred@460297510 after retry RPC after error@853247877) -> (continuation of Deferred@1086618220 after retry RPC after error@1671359015) -> (continuation of Deferred@71275991 after retry RPC after error@1790365050)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.1.18.189:41451
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=28488), Traces: [0ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:44871, [112ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:41451, [124ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:35079, [193ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:44871: Network error: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871, [188ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:41451: Network error: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [650ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:35079: OK, [657ms] delaying RPC due to: Service unavailable: Master config (127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [676ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:44871, [682ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:41451, [683ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:44871: Network error: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871, [684ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:35079, [690ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:41451: Network error: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [693ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:35079: OK, [698ms] delaying RPC due to: Service unavailable: Master config (127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [715ms] refreshing cache from master, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:44871, [719ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:41451, [720ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:44871: Network error: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871, [721ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:35079, [726ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:41451: Network error: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [728ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:35079: OK, [730ms] delaying RPC due to: Service unavailable: Master config (127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [735ms] refreshing cache from master, [735ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:44871, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:41451, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:35079, [739ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:44871: Network error: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871, [744ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:41451: Network error: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [746ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:35079: OK, [747ms] delaying RPC due to: Service unavailable: Master config (127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [775ms] refreshing cache from master, [775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:44871, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:41451, [779ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:44871: Network error: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871, [779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:35079, [784ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:41451: Network error: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [786ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:35079: OK, [787ms] delaying RPC due to: Service unavailable: Master config (127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [795ms] refreshing cache from master, [795ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:44871, [797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:41451, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:35079, [798ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:44871: Network error: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871, [801ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:41451: Network error: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [804ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:35079: OK, [805ms] delaying RPC due to: Service unavailable: Master config (127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [834ms] refreshing cache from master, [835ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:44871, [837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:41451, [838ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:44871: Network error: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871, [840ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:35079, [845ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:41451: Network error: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [848ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:35079: OK, [849ms] delaying RPC due to: Service unavailable: Master config (127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [935ms] refreshing cache from master, [935ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:44871, [938ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:41451, [939ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:35079, [939ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:44871: Network error: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871, [943ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:41451: Network error: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [955ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:35079: OK, [958ms] delaying RPC due to: Service unavailable: Master config (127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [1114ms] refreshing cache from master, [1115ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:44871, [1116ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:41451, [1118ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:35079, [1118ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:44871: Network error: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871, [1123ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:41451: Network error: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [1125ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:35079: OK, [1127ms] delaying RPC due to: Service unavailable: Master config (127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [1514ms] refreshing cache from master, [1515ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:44871, [1517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:41451, [1519ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:35079, [1519ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:44871: Network error: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871, [1522ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:41451: Network error: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [1525ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:35079: OK, [1526ms] delaying RPC due to: Service unavailable: Master config (127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [1895ms] refreshing cache from master, [1895ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:44871, [1897ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:41451, [1898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:35079, [1898ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:44871: Network error: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871, [1902ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:41451: Network error: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [1907ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:35079: OK, [1907ms] delaying RPC due to: Service unavailable: Master config (127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [3795ms] refreshing cache from master, [3796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:44871, [3797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:41451, [3799ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:35079, [3799ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:44871: Network error: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871, [3802ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:41451: Network error: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [3804ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:35079: OK, [3805ms] delaying RPC due to: Service unavailable: Master config (127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [5734ms] refreshing cache from master, [5735ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:44871, [5736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:41451, [5737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:35079, [5738ms] too many traces: truncated at 100 traces, deferred=Deferred@818840767(state=PENDING, result=null, callback=(continuation of Deferred@109207934 after retry RPC after error@38856450) -> (continuation of Deferred@1504326812 after retry RPC after error@486910058) -> (continuation of Deferred@135901989 after retry RPC after error@225721811) -> (continuation of Deferred@91420213 after retry RPC after error@889477129) -> (continuation of Deferred@1891718423 after retry RPC after error@1551332473) -> (continuation of Deferred@1319563932 after retry RPC after error@1509244753) -> (continuation of Deferred@593932437 after retry RPC after error@1154862140) -> (continuation of Deferred@1025438760 after retry RPC after error@1306610541) -> (continuation of Deferred@1536534620 after retry RPC after error@827013950) -> (continuation of Deferred@2067674763 after retry RPC after error@1011069090) -> (continuation of Deferred@1247946814 after retry RPC after error@580413575) -> (continuation of Deferred@672700465 after retry RPC after error@167766832) -> (continuation of Deferred@1376190953 after retry RPC after error@867862398) -> (continuation of Deferred@1594837032 after retry RPC after error@702516882) -> (continuation of Deferred@1136511497 after retry RPC after error@618345591) -> (continuation of Deferred@28337868 after retry RPC after error@1422836598) -> (continuation of Deferred@2039391844 after retry RPC after error@76238254) -> (continuation of Deferred@101703157 after retry RPC after error@620260145) -> (continuation of Deferred@684278860 after retry RPC after error@1995545286) -> (continuation of Deferred@250155416 after retry RPC after error@1764709015) -> (continuation of Deferred@195465434 after retry RPC after error@1912282306) -> (continuation of Deferred@936945332 after retry RPC after error@611381554) -> (continuation of Deferred@460297510 after retry RPC after error@853247877) -> (continuation of Deferred@1086618220 after retry RPC after error@1671359015) -> (continuation of Deferred@71275991 after retry RPC after error@1790365050), errback=(continuation of Deferred@109207934 after retry RPC after error@38856450) -> (continuation of Deferred@1504326812 after retry RPC after error@486910058) -> (continuation of Deferred@135901989 after retry RPC after error@225721811) -> (continuation of Deferred@91420213 after retry RPC after error@889477129) -> (continuation of Deferred@1891718423 after retry RPC after error@1551332473) -> (continuation of Deferred@1319563932 after retry RPC after error@1509244753) -> (continuation of Deferred@593932437 after retry RPC after error@1154862140) -> (continuation of Deferred@1025438760 after retry RPC after error@1306610541) -> (continuation of Deferred@1536534620 after retry RPC after error@827013950) -> (continuation of Deferred@2067674763 after retry RPC after error@1011069090) -> (continuation of Deferred@1247946814 after retry RPC after error@580413575) -> (continuation of Deferred@672700465 after retry RPC after error@167766832) -> (continuation of Deferred@1376190953 after retry RPC after error@867862398) -> (continuation of Deferred@1594837032 after retry RPC after error@702516882) -> (continuation of Deferred@1136511497 after retry RPC after error@618345591) -> (continuation of Deferred@28337868 after retry RPC after error@1422836598) -> (continuation of Deferred@2039391844 after retry RPC after error@76238254) -> (continuation of Deferred@101703157 after retry RPC after error@620260145) -> (continuation of Deferred@684278860 after retry RPC after error@1995545286) -> (continuation of Deferred@250155416 after retry RPC after error@1764709015) -> (continuation of Deferred@195465434 after retry RPC after error@1912282306) -> (continuation of Deferred@936945332 after retry RPC after error@611381554) -> (continuation of Deferred@460297510 after retry RPC after error@853247877) -> (continuation of Deferred@1086618220 after retry RPC after error@1671359015) -> (continuation of Deferred@71275991 after retry RPC after error@1790365050)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.1.18.189:41451
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
21:44:32.604 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
21:44:32.604 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=28488), Traces: [0ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:44871, [112ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:41451, [124ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:35079, [193ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:44871: Network error: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871, [188ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:41451: Network error: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [650ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:35079: OK, [657ms] delaying RPC due to: Service unavailable: Master config (127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [676ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:44871, [682ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:41451, [683ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:44871: Network error: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871, [684ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:35079, [690ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:41451: Network error: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [693ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:35079: OK, [698ms] delaying RPC due to: Service unavailable: Master config (127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [715ms] refreshing cache from master, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:44871, [719ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:41451, [720ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:44871: Network error: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871, [721ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:35079, [726ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:41451: Network error: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [728ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:35079: OK, [730ms] delaying RPC due to: Service unavailable: Master config (127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [735ms] refreshing cache from master, [735ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:44871, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:41451, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:35079, [739ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:44871: Network error: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871, [744ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:41451: Network error: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [746ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:35079: OK, [747ms] delaying RPC due to: Service unavailable: Master config (127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [775ms] refreshing cache from master, [775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:44871, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:41451, [779ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:44871: Network error: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871, [779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:35079, [784ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:41451: Network error: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [786ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:35079: OK, [787ms] delaying RPC due to: Service unavailable: Master config (127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [795ms] refreshing cache from master, [795ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:44871, [797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:41451, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:35079, [798ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:44871: Network error: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871, [801ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:41451: Network error: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [804ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:35079: OK, [805ms] delaying RPC due to: Service unavailable: Master config (127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [834ms] refreshing cache from master, [835ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:44871, [837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:41451, [838ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:44871: Network error: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871, [840ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:35079, [845ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:41451: Network error: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [848ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:35079: OK, [849ms] delaying RPC due to: Service unavailable: Master config (127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [935ms] refreshing cache from master, [935ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:44871, [938ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:41451, [939ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:35079, [939ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:44871: Network error: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871, [943ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:41451: Network error: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [955ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:35079: OK, [958ms] delaying RPC due to: Service unavailable: Master config (127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [1114ms] refreshing cache from master, [1115ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:44871, [1116ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:41451, [1118ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:35079, [1118ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:44871: Network error: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871, [1123ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:41451: Network error: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [1125ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:35079: OK, [1127ms] delaying RPC due to: Service unavailable: Master config (127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [1514ms] refreshing cache from master, [1515ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:44871, [1517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:41451, [1519ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:35079, [1519ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:44871: Network error: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871, [1522ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:41451: Network error: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [1525ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:35079: OK, [1526ms] delaying RPC due to: Service unavailable: Master config (127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [1895ms] refreshing cache from master, [1895ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:44871, [1897ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:41451, [1898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:35079, [1898ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:44871: Network error: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871, [1902ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:41451: Network error: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [1907ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:35079: OK, [1907ms] delaying RPC due to: Service unavailable: Master config (127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [3795ms] refreshing cache from master, [3796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:44871, [3797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:41451, [3799ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:35079, [3799ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:44871: Network error: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871, [3802ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:41451: Network error: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [3804ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:35079: OK, [3805ms] delaying RPC due to: Service unavailable: Master config (127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [5734ms] refreshing cache from master, [5735ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:44871, [5736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:41451, [5737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:35079, [5738ms] too many traces: truncated at 100 traces, deferred=Deferred@818840767(state=PENDING, result=null, callback=(continuation of Deferred@109207934 after retry RPC after error@38856450) -> (continuation of Deferred@1504326812 after retry RPC after error@486910058) -> (continuation of Deferred@135901989 after retry RPC after error@225721811) -> (continuation of Deferred@91420213 after retry RPC after error@889477129) -> (continuation of Deferred@1891718423 after retry RPC after error@1551332473) -> (continuation of Deferred@1319563932 after retry RPC after error@1509244753) -> (continuation of Deferred@593932437 after retry RPC after error@1154862140) -> (continuation of Deferred@1025438760 after retry RPC after error@1306610541) -> (continuation of Deferred@1536534620 after retry RPC after error@827013950) -> (continuation of Deferred@2067674763 after retry RPC after error@1011069090) -> (continuation of Deferred@1247946814 after retry RPC after error@580413575) -> (continuation of Deferred@672700465 after retry RPC after error@167766832) -> (continuation of Deferred@1376190953 after retry RPC after error@867862398) -> (continuation of Deferred@1594837032 after retry RPC after error@702516882) -> (continuation of Deferred@1136511497 after retry RPC after error@618345591) -> (continuation of Deferred@28337868 after retry RPC after error@1422836598) -> (continuation of Deferred@2039391844 after retry RPC after error@76238254) -> (continuation of Deferred@101703157 after retry RPC after error@620260145) -> (continuation of Deferred@684278860 after retry RPC after error@1995545286) -> (continuation of Deferred@250155416 after retry RPC after error@1764709015) -> (continuation of Deferred@195465434 after retry RPC after error@1912282306) -> (continuation of Deferred@936945332 after retry RPC after error@611381554) -> (continuation of Deferred@460297510 after retry RPC after error@853247877) -> (continuation of Deferred@1086618220 after retry RPC after error@1671359015) -> (continuation of Deferred@71275991 after retry RPC after error@1790365050), errback=(continuation of Deferred@109207934 after retry RPC after error@38856450) -> (continuation of Deferred@1504326812 after retry RPC after error@486910058) -> (continuation of Deferred@135901989 after retry RPC after error@225721811) -> (continuation of Deferred@91420213 after retry RPC after error@889477129) -> (continuation of Deferred@1891718423 after retry RPC after error@1551332473) -> (continuation of Deferred@1319563932 after retry RPC after error@1509244753) -> (continuation of Deferred@593932437 after retry RPC after error@1154862140) -> (continuation of Deferred@1025438760 after retry RPC after error@1306610541) -> (continuation of Deferred@1536534620 after retry RPC after error@827013950) -> (continuation of Deferred@2067674763 after retry RPC after error@1011069090) -> (continuation of Deferred@1247946814 after retry RPC after error@580413575) -> (continuation of Deferred@672700465 after retry RPC after error@167766832) -> (continuation of Deferred@1376190953 after retry RPC after error@867862398) -> (continuation of Deferred@1594837032 after retry RPC after error@702516882) -> (continuation of Deferred@1136511497 after retry RPC after error@618345591) -> (continuation of Deferred@28337868 after retry RPC after error@1422836598) -> (continuation of Deferred@2039391844 after retry RPC after error@76238254) -> (continuation of Deferred@101703157 after retry RPC after error@620260145) -> (continuation of Deferred@684278860 after retry RPC after error@1995545286) -> (continuation of Deferred@250155416 after retry RPC after error@1764709015) -> (continuation of Deferred@195465434 after retry RPC after error@1912282306) -> (continuation of Deferred@936945332 after retry RPC after error@611381554) -> (continuation of Deferred@460297510 after retry RPC after error@853247877) -> (continuation of Deferred@1086618220 after retry RPC after error@1671359015) -> (continuation of Deferred@71275991 after retry RPC after error@1790365050)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.1.18.189:41451
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=28488), Traces: [0ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:44871, [112ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:41451, [124ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:35079, [193ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:44871: Network error: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871, [188ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:41451: Network error: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [650ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:35079: OK, [657ms] delaying RPC due to: Service unavailable: Master config (127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [676ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:44871, [682ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:41451, [683ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:44871: Network error: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871, [684ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:35079, [690ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:41451: Network error: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [693ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:35079: OK, [698ms] delaying RPC due to: Service unavailable: Master config (127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [715ms] refreshing cache from master, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:44871, [719ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:41451, [720ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:44871: Network error: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871, [721ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:35079, [726ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:41451: Network error: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [728ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:35079: OK, [730ms] delaying RPC due to: Service unavailable: Master config (127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [735ms] refreshing cache from master, [735ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:44871, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:41451, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:35079, [739ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:44871: Network error: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871, [744ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:41451: Network error: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [746ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:35079: OK, [747ms] delaying RPC due to: Service unavailable: Master config (127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [775ms] refreshing cache from master, [775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:44871, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:41451, [779ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:44871: Network error: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871, [779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:35079, [784ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:41451: Network error: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [786ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:35079: OK, [787ms] delaying RPC due to: Service unavailable: Master config (127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [795ms] refreshing cache from master, [795ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:44871, [797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:41451, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:35079, [798ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:44871: Network error: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871, [801ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:41451: Network error: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [804ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:35079: OK, [805ms] delaying RPC due to: Service unavailable: Master config (127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [834ms] refreshing cache from master, [835ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:44871, [837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:41451, [838ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:44871: Network error: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871, [840ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:35079, [845ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:41451: Network error: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [848ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:35079: OK, [849ms] delaying RPC due to: Service unavailable: Master config (127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [935ms] refreshing cache from master, [935ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:44871, [938ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:41451, [939ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:35079, [939ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:44871: Network error: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871, [943ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:41451: Network error: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [955ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:35079: OK, [958ms] delaying RPC due to: Service unavailable: Master config (127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [1114ms] refreshing cache from master, [1115ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:44871, [1116ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:41451, [1118ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:35079, [1118ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:44871: Network error: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871, [1123ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:41451: Network error: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [1125ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:35079: OK, [1127ms] delaying RPC due to: Service unavailable: Master config (127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [1514ms] refreshing cache from master, [1515ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:44871, [1517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:41451, [1519ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:35079, [1519ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:44871: Network error: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871, [1522ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:41451: Network error: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [1525ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:35079: OK, [1526ms] delaying RPC due to: Service unavailable: Master config (127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [1895ms] refreshing cache from master, [1895ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:44871, [1897ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:41451, [1898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:35079, [1898ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:44871: Network error: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871, [1902ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:41451: Network error: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [1907ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:35079: OK, [1907ms] delaying RPC due to: Service unavailable: Master config (127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [3795ms] refreshing cache from master, [3796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:44871, [3797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:41451, [3799ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:35079, [3799ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:44871: Network error: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871, [3802ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:41451: Network error: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [3804ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:35079: OK, [3805ms] delaying RPC due to: Service unavailable: Master config (127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451, [5734ms] refreshing cache from master, [5735ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:44871, [5736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:41451, [5737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:35079, [5738ms] too many traces: truncated at 100 traces, deferred=Deferred@818840767(state=PENDING, result=null, callback=(continuation of Deferred@109207934 after retry RPC after error@38856450) -> (continuation of Deferred@1504326812 after retry RPC after error@486910058) -> (continuation of Deferred@135901989 after retry RPC after error@225721811) -> (continuation of Deferred@91420213 after retry RPC after error@889477129) -> (continuation of Deferred@1891718423 after retry RPC after error@1551332473) -> (continuation of Deferred@1319563932 after retry RPC after error@1509244753) -> (continuation of Deferred@593932437 after retry RPC after error@1154862140) -> (continuation of Deferred@1025438760 after retry RPC after error@1306610541) -> (continuation of Deferred@1536534620 after retry RPC after error@827013950) -> (continuation of Deferred@2067674763 after retry RPC after error@1011069090) -> (continuation of Deferred@1247946814 after retry RPC after error@580413575) -> (continuation of Deferred@672700465 after retry RPC after error@167766832) -> (continuation of Deferred@1376190953 after retry RPC after error@867862398) -> (continuation of Deferred@1594837032 after retry RPC after error@702516882) -> (continuation of Deferred@1136511497 after retry RPC after error@618345591) -> (continuation of Deferred@28337868 after retry RPC after error@1422836598) -> (continuation of Deferred@2039391844 after retry RPC after error@76238254) -> (continuation of Deferred@101703157 after retry RPC after error@620260145) -> (continuation of Deferred@684278860 after retry RPC after error@1995545286) -> (continuation of Deferred@250155416 after retry RPC after error@1764709015) -> (continuation of Deferred@195465434 after retry RPC after error@1912282306) -> (continuation of Deferred@936945332 after retry RPC after error@611381554) -> (continuation of Deferred@460297510 after retry RPC after error@853247877) -> (continuation of Deferred@1086618220 after retry RPC after error@1671359015) -> (continuation of Deferred@71275991 after retry RPC after error@1790365050), errback=(continuation of Deferred@109207934 after retry RPC after error@38856450) -> (continuation of Deferred@1504326812 after retry RPC after error@486910058) -> (continuation of Deferred@135901989 after retry RPC after error@225721811) -> (continuation of Deferred@91420213 after retry RPC after error@889477129) -> (continuation of Deferred@1891718423 after retry RPC after error@1551332473) -> (continuation of Deferred@1319563932 after retry RPC after error@1509244753) -> (continuation of Deferred@593932437 after retry RPC after error@1154862140) -> (continuation of Deferred@1025438760 after retry RPC after error@1306610541) -> (continuation of Deferred@1536534620 after retry RPC after error@827013950) -> (continuation of Deferred@2067674763 after retry RPC after error@1011069090) -> (continuation of Deferred@1247946814 after retry RPC after error@580413575) -> (continuation of Deferred@672700465 after retry RPC after error@167766832) -> (continuation of Deferred@1376190953 after retry RPC after error@867862398) -> (continuation of Deferred@1594837032 after retry RPC after error@702516882) -> (continuation of Deferred@1136511497 after retry RPC after error@618345591) -> (continuation of Deferred@28337868 after retry RPC after error@1422836598) -> (continuation of Deferred@2039391844 after retry RPC after error@76238254) -> (continuation of Deferred@101703157 after retry RPC after error@620260145) -> (continuation of Deferred@684278860 after retry RPC after error@1995545286) -> (continuation of Deferred@250155416 after retry RPC after error@1764709015) -> (continuation of Deferred@195465434 after retry RPC after error@1912282306) -> (continuation of Deferred@936945332 after retry RPC after error@611381554) -> (continuation of Deferred@460297510 after retry RPC after error@853247877) -> (continuation of Deferred@1086618220 after retry RPC after error@1671359015) -> (continuation of Deferred@71275991 after retry RPC after error@1790365050)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.1.18.190:44871,127.1.18.189:41451,127.1.18.188:35079) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:44871(127.1.18.190:44871): Connection refused: /127.1.18.190:44871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:41451(127.1.18.189:41451): Connection refused: /127.1.18.189:41451
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.1.18.189:41451
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)