05:19:09.625 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
05:19:09.638 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskHSJIN2/test-tmp
05:19:09.640 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
05:19:09.643 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskHSJIN2/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
05:19:09.932 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
05:19:10.077 [DEBUG - main] (MiniKuduCluster.java:195) Response:
05:19:10.078 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
05:19:10.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:10.082350 7576 mini_oidc.cc:152] Starting JWKS server
05:19:10.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:10.086176 7576 webserver.cc:460] Webserver started at http://127.0.0.1:46607/ using document root <none> and password file <none>
05:19:10.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:10.086735 7576 mini_oidc.cc:189] Starting OIDC Discovery server
05:19:10.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:10.089668 7576 webserver.cc:460] Webserver started at http://127.0.0.1:42703/ using document root <none> and password file <none>
05:19:10.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-28T05:19:10Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
05:19:10.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-28T05:19:10Z Disabled control of system clock
05:19:10.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:10.140444 7576 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskHSJIN2/build/tsan/bin/kudu
05:19:10.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskHSJIN2/build/tsan/bin/kudu
05:19:10.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/master-0/wal
05:19:10.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/master-0/data
05:19:10.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:19:10.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:19:10.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:19:10.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:19:10.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:19:10.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:19:10.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/master-0/logs
05:19:10.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/master-0/data/info.pb
05:19:10.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:19:10.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:19:10.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:19:10.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:19:10.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:19:10.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:19:10.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:19:10.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:19:10.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:19:10.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:19:10.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:19:10.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:19:10.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.102.62:43223
05:19:10.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.102.62
05:19:10.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:19:10.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.102.20:44503
05:19:10.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:19:10.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:19:10.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:19:10.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:19:10.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.102.62:43223,127.7.102.61:35105,127.7.102.60:35175
05:19:10.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:19:10.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46607/jwks.json with env {}
05:19:10.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:10.446178 7588 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:19:10.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:10.446890 7588 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:19:10.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:10.447654 7588 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:19:10.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:10.478181 7588 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:19:10.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:10.478583 7588 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:19:10.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:10.478904 7588 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:19:10.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:10.479256 7588 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:19:10.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:10.479717 7588 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:19:10.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:10.480039 7588 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46607/jwks.json
05:19:10.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:10.514459 7588 master_runner.cc:386] Master server non-default flags:
05:19:10.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:19:10.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.102.20:44503
05:19:10.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:19:10.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:19:10.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/master-0/data
05:19:10.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/master-0/wal
05:19:10.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:19:10.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.102.62:43223,127.7.102.61:35105,127.7.102.60:35175
05:19:10.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:19:10.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:19:10.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:19:10.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.102.62:43223
05:19:10.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:19:10.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:19:10.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:19:10.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46607/jwks.json
05:19:10.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:19:10.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:19:10.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/master-0/data/info.pb
05:19:10.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.102.62
05:19:10.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:19:10.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:19:10.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:19:10.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:19:10.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:19:10.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:19:10.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/master-0/logs
05:19:10.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:19:10.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:19:10.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:19:10.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:19:10.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:19:10.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:19:10.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:19:10.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 04:35:01 UTC on ccycloud.quasar-alllgi.root.comops.site
05:19:10.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676563
05:19:10.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:19:10.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:10.516083 7588 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:19:10.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:10.518273 7588 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:19:10.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:10.538589 7594 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:19:11.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:10.538625 7595 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:19:11.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:10.540493 7588 server_base.cc:1034] running on GCE node
05:19:11.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:10.538861 7597 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:19:11.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:11.799674 7588 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:19:11.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:11.803462 7588 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:19:11.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:11.804982 7588 hybrid_clock.cc:648] HybridClock initialized: now 1790572751804946 us; error 56 us; skew 500 ppm
05:19:11.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:11.805895 7588 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:19:11.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:11.820719 7588 webserver.cc:460] Webserver started at http://127.7.102.62:41079/ using document root <none> and password file <none>
05:19:11.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:11.821663 7588 fs_manager.cc:359] Metadata directory not provided
05:19:11.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:11.821851 7588 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:19:11.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:11.822233 7588 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:19:11.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:11.826967 7588 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/master-0/data/instance:
05:19:11.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "46ab27132ad44da4937b52e36a4f462e"
05:19:11.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 05:19:11 on dist-test-slave-lhv2"
05:19:11.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:11.828138 7588 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/master-0/wal/instance:
05:19:11.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "46ab27132ad44da4937b52e36a4f462e"
05:19:11.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 05:19:11 on dist-test-slave-lhv2"
05:19:11.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:11.835574 7588 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
05:19:11.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:11.842594 7604 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:19:11.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:11.843771 7588 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.003s sys 0.000s
05:19:11.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:11.844120 7588 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/master-0/data,/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/master-0/wal
05:19:11.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "46ab27132ad44da4937b52e36a4f462e"
05:19:11.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 05:19:11 on dist-test-slave-lhv2"
05:19:11.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:11.844521 7588 fs_report.cc:383] FS layout report
05:19:11.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:19:11.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/master-0/wal
05:19:11.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/master-0/wal
05:19:11.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/master-0/data/data
05:19:11.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:19:11.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:19:11.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:19:11.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:19:11.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:19:11.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:19:11.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:19:11.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:19:11.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:19:11.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:19:11.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:19:11.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:11.896046 7588 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:19:11.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:11.897482 7588 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:19:11.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:11.897884 7588 kserver.cc:163] Server-wide thread pool size limit: 3276
05:19:11.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:11.976145 7588 rpc_server.cc:307] RPC server started. Bound to: 127.7.102.62:43223
05:19:11.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:11.976184 7655 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.102.62:43223 every 8 connection(s)
05:19:11.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:11.978873 7588 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/master-0/data/info.pb
05:19:11.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:11.984405 7656 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
05:19:11.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:11.988229 7576 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskHSJIN2/build/tsan/bin/kudu as pid 7588
05:19:11.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:11.988606 7576 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/master-0/wal/instance
05:19:11.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:11.991995 7576 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskHSJIN2/build/tsan/bin/kudu
05:19:11.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskHSJIN2/build/tsan/bin/kudu
05:19:11.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/master-1/wal
05:19:11.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/master-1/data
05:19:11.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:19:11.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:19:11.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:19:11.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:19:11.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:19:11.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:19:11.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/master-1/logs
05:19:11.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/master-1/data/info.pb
05:19:11.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:19:11.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:19:11.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:19:11.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:19:11.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:19:11.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:19:11.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:19:11.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:19:11.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:19:11.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:19:11.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:19:11.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:19:11.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.102.61:35105
05:19:11.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.102.61
05:19:11.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:19:11.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.102.20:44503
05:19:11.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:19:11.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:19:11.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:19:11.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:19:11.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.102.62:43223,127.7.102.61:35105,127.7.102.60:35175
05:19:11.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:19:11.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46607/jwks.json with env {}
05:19:12.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:11.998710 7656 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.102.62" port: 43223 } has no permanent_uuid. Determining permanent_uuid...
05:19:12.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:12.028565 7656 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.102.61" port: 35105 } has no permanent_uuid. Determining permanent_uuid...
05:19:12.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:12.031131 7608 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.102.61:35105: connect: Connection refused (error 111)
05:19:12.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:12.035327 7656 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.102.61:35105: Network error: Client connection negotiation failed: client connection to 127.7.102.61:35105: connect: Connection refused (error 111)
05:19:12.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:12.084774 7656 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.102.61" port: 35105 } attempt: 1
05:19:12.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:12.089986 7656 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.102.61:35105: Network error: Client connection negotiation failed: client connection to 127.7.102.61:35105: connect: Connection refused (error 111)
05:19:12.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:12.158460 7656 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.102.61" port: 35105 } attempt: 2
05:19:12.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:12.162940 7656 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.102.61:35105: Network error: Client connection negotiation failed: client connection to 127.7.102.61:35105: connect: Connection refused (error 111)
05:19:12.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:12.254422 7656 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.102.61" port: 35105 } attempt: 3
05:19:12.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:12.258065 7656 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.102.61:35105: Network error: Client connection negotiation failed: client connection to 127.7.102.61:35105: connect: Connection refused (error 111)
05:19:12.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:12.309629 7658 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:19:12.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:12.310177 7658 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:19:12.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:12.310858 7658 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:19:12.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:12.342854 7658 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:19:12.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:12.343134 7658 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:19:12.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:12.343420 7658 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:19:12.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:12.343617 7658 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:19:12.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:12.343791 7658 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:19:12.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:12.343918 7658 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46607/jwks.json
05:19:12.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:12.379653 7658 master_runner.cc:386] Master server non-default flags:
05:19:12.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:19:12.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.102.20:44503
05:19:12.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:19:12.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:19:12.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/master-1/data
05:19:12.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/master-1/wal
05:19:12.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:19:12.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.102.62:43223,127.7.102.61:35105,127.7.102.60:35175
05:19:12.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:19:12.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:19:12.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:19:12.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.102.61:35105
05:19:12.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:19:12.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:19:12.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:19:12.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46607/jwks.json
05:19:12.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:19:12.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:19:12.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/master-1/data/info.pb
05:19:12.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.102.61
05:19:12.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:19:12.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:19:12.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:19:12.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:19:12.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:19:12.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:19:12.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/master-1/logs
05:19:12.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:19:12.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:19:12.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:19:12.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:19:12.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:19:12.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:19:12.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:19:12.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 04:35:01 UTC on ccycloud.quasar-alllgi.root.comops.site
05:19:12.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676563
05:19:12.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:19:12.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:12.380805 7658 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:19:12.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:12.382952 7658 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:19:12.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:12.401451 7656 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.102.61" port: 35105 } attempt: 4
05:19:12.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:12.401947 7670 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:19:12.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:12.405987 7656 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.102.61:35105: Network error: Client connection negotiation failed: client connection to 127.7.102.61:35105: connect: Connection refused (error 111)
05:19:12.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:12.705432 7656 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.102.61" port: 35105 } attempt: 5
05:19:12.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:12.711108 7656 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.102.61:35105: Network error: Client connection negotiation failed: client connection to 127.7.102.61:35105: connect: Connection refused (error 111)
05:19:13.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:13.258639 7656 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.102.61" port: 35105 } attempt: 6
05:19:13.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:13.266831 7656 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.102.61:35105: Network error: Client connection negotiation failed: client connection to 127.7.102.61:35105: connect: Connection refused (error 111)
05:19:13.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:12.402000 7668 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:19:13.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:12.404204 7667 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:19:13.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:13.609611 7669 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
05:19:13.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:13.609730 7658 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
05:19:13.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:13.614195 7658 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:19:13.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:13.617600 7658 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:19:13.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:13.619107 7658 hybrid_clock.cc:648] HybridClock initialized: now 1790572753619015 us; error 63 us; skew 500 ppm
05:19:13.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:13.619959 7658 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:19:13.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:13.626901 7658 webserver.cc:460] Webserver started at http://127.7.102.61:33213/ using document root <none> and password file <none>
05:19:13.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:13.627877 7658 fs_manager.cc:359] Metadata directory not provided
05:19:13.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:13.628067 7658 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:19:13.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:13.628443 7658 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:19:13.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:13.633296 7658 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/master-1/data/instance:
05:19:13.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5645663ae6034d8d9f14e74dfb612ec3"
05:19:13.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 05:19:13 on dist-test-slave-lhv2"
05:19:13.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:13.634521 7658 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/master-1/wal/instance:
05:19:13.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5645663ae6034d8d9f14e74dfb612ec3"
05:19:13.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 05:19:13 on dist-test-slave-lhv2"
05:19:13.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:13.642028 7658 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.004s
05:19:13.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:13.647837 7679 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:19:13.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:13.648984 7658 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
05:19:13.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:13.649266 7658 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/master-1/data,/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/master-1/wal
05:19:13.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5645663ae6034d8d9f14e74dfb612ec3"
05:19:13.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 05:19:13 on dist-test-slave-lhv2"
05:19:13.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:13.649552 7658 fs_report.cc:383] FS layout report
05:19:13.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:19:13.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/master-1/wal
05:19:13.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/master-1/wal
05:19:13.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/master-1/data/data
05:19:13.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:19:13.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:19:13.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:19:13.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:19:13.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:19:13.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:19:13.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:19:13.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:19:13.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:19:13.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:19:13.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:19:13.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:13.689445 7658 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:19:13.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:13.690930 7658 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:19:13.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:13.691476 7658 kserver.cc:163] Server-wide thread pool size limit: 3276
05:19:13.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:13.771103 7658 rpc_server.cc:307] RPC server started. Bound to: 127.7.102.61:35105
05:19:13.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:13.771247 7730 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.102.61:35105 every 8 connection(s)
05:19:13.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:13.773794 7658 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/master-1/data/info.pb
05:19:13.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:13.779549 7731 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
05:19:13.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:13.779492 7576 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskHSJIN2/build/tsan/bin/kudu as pid 7658
05:19:13.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:13.779863 7576 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/master-1/wal/instance
05:19:13.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:13.782019 7576 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskHSJIN2/build/tsan/bin/kudu
05:19:13.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskHSJIN2/build/tsan/bin/kudu
05:19:13.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/master-2/wal
05:19:13.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/master-2/data
05:19:13.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:19:13.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:19:13.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:19:13.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:19:13.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:19:13.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:19:13.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/master-2/logs
05:19:13.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/master-2/data/info.pb
05:19:13.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:19:13.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:19:13.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:19:13.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:19:13.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:19:13.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:19:13.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:19:13.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:19:13.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:19:13.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:19:13.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:19:13.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:19:13.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.102.60:35175
05:19:13.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.102.60
05:19:13.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:19:13.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.102.20:44503
05:19:13.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:19:13.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:19:13.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:19:13.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:19:13.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.102.62:43223,127.7.102.61:35105,127.7.102.60:35175
05:19:13.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:19:13.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46607/jwks.json with env {}
05:19:13.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:13.796660 7731 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.102.62" port: 43223 } has no permanent_uuid. Determining permanent_uuid...
05:19:13.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:13.825304 7731 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.102.61" port: 35105 } has no permanent_uuid. Determining permanent_uuid...
05:19:13.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:13.844075 7731 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.102.60" port: 35175 } has no permanent_uuid. Determining permanent_uuid...
05:19:13.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:13.847234 7680 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.102.60:35175: connect: Connection refused (error 111)
05:19:13.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:13.854192 7731 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.102.60:35175: Network error: Client connection negotiation failed: client connection to 127.7.102.60:35175: connect: Connection refused (error 111)
05:19:13.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:13.903712 7731 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.102.60" port: 35175 } attempt: 1
05:19:13.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:13.907874 7731 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.102.60:35175: Network error: Client connection negotiation failed: client connection to 127.7.102.60:35175: connect: Connection refused (error 111)
05:19:13.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:13.976392 7731 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.102.60" port: 35175 } attempt: 2
05:19:13.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:13.980633 7731 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.102.60:35175: Network error: Client connection negotiation failed: client connection to 127.7.102.60:35175: connect: Connection refused (error 111)
05:19:14.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:14.072108 7731 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.102.60" port: 35175 } attempt: 3
05:19:14.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:14.076109 7731 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.102.60:35175: Network error: Client connection negotiation failed: client connection to 127.7.102.60:35175: connect: Connection refused (error 111)
05:19:14.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:14.138845 7733 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:19:14.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:14.139446 7733 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:19:14.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:14.139909 7733 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:19:14.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:14.172547 7733 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:19:14.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:14.172838 7733 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:19:14.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:14.173132 7733 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:19:14.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:14.173341 7733 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:19:14.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:14.173525 7733 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:19:14.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:14.173774 7733 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46607/jwks.json
05:19:14.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:14.209152 7733 master_runner.cc:386] Master server non-default flags:
05:19:14.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:19:14.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.102.20:44503
05:19:14.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:19:14.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:19:14.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/master-2/data
05:19:14.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/master-2/wal
05:19:14.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:19:14.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.102.62:43223,127.7.102.61:35105,127.7.102.60:35175
05:19:14.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:19:14.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:19:14.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:19:14.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.102.60:35175
05:19:14.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:19:14.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:19:14.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:19:14.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46607/jwks.json
05:19:14.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:19:14.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:19:14.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/master-2/data/info.pb
05:19:14.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.102.60
05:19:14.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:19:14.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:19:14.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:19:14.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:19:14.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:19:14.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:19:14.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/master-2/logs
05:19:14.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:19:14.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:19:14.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:19:14.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:19:14.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:19:14.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:19:14.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:19:14.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 04:35:01 UTC on ccycloud.quasar-alllgi.root.comops.site
05:19:14.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676563
05:19:14.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:19:14.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:14.210398 7733 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:19:14.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:14.212419 7733 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:19:14.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:14.219614 7731 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.102.60" port: 35175 } attempt: 4
05:19:14.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:14.226097 7731 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.102.60:35175: Network error: Client connection negotiation failed: client connection to 127.7.102.60:35175: connect: Connection refused (error 111)
05:19:14.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:14.234124 7744 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:19:14.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:14.327306 7656 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.102.61" port: 35105 } attempt: 7
05:19:14.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:14.341281 7656 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.102.60" port: 35175 } has no permanent_uuid. Determining permanent_uuid...
05:19:14.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:14.347781 7656 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.102.60:35175: Network error: Client connection negotiation failed: client connection to 127.7.102.60:35175: connect: Connection refused (error 111)
05:19:14.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:14.406210 7656 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.102.60" port: 35175 } attempt: 1
05:19:14.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:14.410634 7656 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.102.60:35175: Network error: Client connection negotiation failed: client connection to 127.7.102.60:35175: connect: Connection refused (error 111)
05:19:14.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:14.492054 7656 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.102.60" port: 35175 } attempt: 2
05:19:14.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:14.496564 7656 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.102.60:35175: Network error: Client connection negotiation failed: client connection to 127.7.102.60:35175: connect: Connection refused (error 111)
05:19:14.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:14.525604 7731 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.102.60" port: 35175 } attempt: 5
05:19:14.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:14.530192 7731 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.102.60:35175: Network error: Client connection negotiation failed: client connection to 127.7.102.60:35175: connect: Connection refused (error 111)
05:19:14.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:14.581975 7656 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.102.60" port: 35175 } attempt: 3
05:19:14.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:14.586724 7656 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.102.60:35175: Network error: Client connection negotiation failed: client connection to 127.7.102.60:35175: connect: Connection refused (error 111)
05:19:14.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:14.727250 7656 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.102.60" port: 35175 } attempt: 4
05:19:14.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:14.731901 7656 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.102.60:35175: Network error: Client connection negotiation failed: client connection to 127.7.102.60:35175: connect: Connection refused (error 111)
05:19:15.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:15.015352 7656 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.102.60" port: 35175 } attempt: 5
05:19:15.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:15.019579 7656 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.102.60:35175: Network error: Client connection negotiation failed: client connection to 127.7.102.60:35175: connect: Connection refused (error 111)
05:19:15.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:15.077605 7731 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.102.60" port: 35175 } attempt: 6
05:19:15.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:15.085354 7731 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.102.60:35175: Network error: Client connection negotiation failed: client connection to 127.7.102.60:35175: connect: Connection refused (error 111)
05:19:15.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:14.235494 7746 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:19:15.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:14.238092 7733 server_base.cc:1034] running on GCE node
05:19:15.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:14.241130 7743 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:19:15.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:15.464159 7733 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:19:15.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:15.466777 7733 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:19:15.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:15.468168 7733 hybrid_clock.cc:648] HybridClock initialized: now 1790572755468143 us; error 43 us; skew 500 ppm
05:19:15.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:15.469192 7733 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:19:15.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:15.476009 7733 webserver.cc:460] Webserver started at http://127.7.102.60:34921/ using document root <none> and password file <none>
05:19:15.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:15.477005 7733 fs_manager.cc:359] Metadata directory not provided
05:19:15.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:15.477240 7733 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:19:15.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:15.477816 7733 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:19:15.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:15.482699 7733 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/master-2/data/instance:
05:19:15.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1b6f87ff43404fec9ea1755cafe56b55"
05:19:15.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 05:19:15 on dist-test-slave-lhv2"
05:19:15.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:15.483906 7733 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/master-2/wal/instance:
05:19:15.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1b6f87ff43404fec9ea1755cafe56b55"
05:19:15.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 05:19:15 on dist-test-slave-lhv2"
05:19:15.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:15.491926 7733 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
05:19:15.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:15.497340 7757 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:19:15.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:15.498422 7733 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.002s
05:19:15.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:15.498821 7733 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/master-2/data,/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/master-2/wal
05:19:15.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1b6f87ff43404fec9ea1755cafe56b55"
05:19:15.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 05:19:15 on dist-test-slave-lhv2"
05:19:15.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:15.499239 7733 fs_report.cc:383] FS layout report
05:19:15.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:19:15.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/master-2/wal
05:19:15.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/master-2/wal
05:19:15.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/master-2/data/data
05:19:15.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:19:15.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:19:15.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:19:15.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:19:15.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:19:15.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:19:15.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:19:15.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:19:15.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:19:15.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:19:15.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:19:15.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:15.536854 7733 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:19:15.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:15.538853 7733 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:19:15.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:15.539409 7733 kserver.cc:163] Server-wide thread pool size limit: 3276
05:19:15.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:15.571960 7656 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.102.60" port: 35175 } attempt: 6
05:19:15.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:15.581079 7656 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.102.60:35175: Network error: Client connection negotiation failed: client connection to 127.7.102.60:35175: connect: Connection refused (error 111)
05:19:15.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:15.623453 7733 rpc_server.cc:307] RPC server started. Bound to: 127.7.102.60:35175
05:19:15.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:15.623562 7810 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.102.60:35175 every 8 connection(s)
05:19:15.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:15.626034 7733 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/master-2/data/info.pb
05:19:15.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:15.632346 7811 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
05:19:15.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:15.633728 7576 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskHSJIN2/build/tsan/bin/kudu as pid 7733
05:19:15.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:15.634186 7576 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/master-2/wal/instance
05:19:15.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:15.650972 7811 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.102.62" port: 43223 } has no permanent_uuid. Determining permanent_uuid...
05:19:15.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:15.684636 7811 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.102.61" port: 35105 } has no permanent_uuid. Determining permanent_uuid...
05:19:15.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:15.696527 7811 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.102.60" port: 35175 } has no permanent_uuid. Determining permanent_uuid...
05:19:15.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:15.714986 7811 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1b6f87ff43404fec9ea1755cafe56b55: Bootstrap starting.
05:19:15.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:15.720968 7811 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 1b6f87ff43404fec9ea1755cafe56b55: Neither blocks nor log segments found. Creating new log.
05:19:15.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:15.724205 7811 log.cc:824] T 00000000000000000000000000000000 P 1b6f87ff43404fec9ea1755cafe56b55: Log is configured to *not* fsync() on all Append() calls
05:19:15.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:15.729069 7811 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1b6f87ff43404fec9ea1755cafe56b55: No bootstrap required, opened a new log
05:19:15.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:15.751632 7811 raft_consensus.cc:348] T 00000000000000000000000000000000 P 1b6f87ff43404fec9ea1755cafe56b55 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "46ab27132ad44da4937b52e36a4f462e" member_type: VOTER last_known_addr { host: "127.7.102.62" port: 43223 } } peers { permanent_uuid: "5645663ae6034d8d9f14e74dfb612ec3" member_type: VOTER last_known_addr { host: "127.7.102.61" port: 35105 } } peers { permanent_uuid: "1b6f87ff43404fec9ea1755cafe56b55" member_type: VOTER last_known_addr { host: "127.7.102.60" port: 35175 } }
05:19:15.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:15.752290 7811 raft_consensus.cc:374] T 00000000000000000000000000000000 P 1b6f87ff43404fec9ea1755cafe56b55 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
05:19:15.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:15.752544 7811 raft_consensus.cc:729] T 00000000000000000000000000000000 P 1b6f87ff43404fec9ea1755cafe56b55 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1b6f87ff43404fec9ea1755cafe56b55, State: Initialized, Role: FOLLOWER
05:19:15.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:15.753453 7811 consensus_queue.cc:260] T 00000000000000000000000000000000 P 1b6f87ff43404fec9ea1755cafe56b55 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "46ab27132ad44da4937b52e36a4f462e" member_type: VOTER last_known_addr { host: "127.7.102.62" port: 43223 } } peers { permanent_uuid: "5645663ae6034d8d9f14e74dfb612ec3" member_type: VOTER last_known_addr { host: "127.7.102.61" port: 35105 } } peers { permanent_uuid: "1b6f87ff43404fec9ea1755cafe56b55" member_type: VOTER last_known_addr { host: "127.7.102.60" port: 35175 } }
05:19:15.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:15.757042 7820 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1b6f87ff43404fec9ea1755cafe56b55 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "46ab27132ad44da4937b52e36a4f462e" member_type: VOTER last_known_addr { host: "127.7.102.62" port: 43223 } } peers { permanent_uuid: "5645663ae6034d8d9f14e74dfb612ec3" member_type: VOTER last_known_addr { host: "127.7.102.61" port: 35105 } } peers { permanent_uuid: "1b6f87ff43404fec9ea1755cafe56b55" member_type: VOTER last_known_addr { host: "127.7.102.60" port: 35175 } } }
05:19:15.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:15.758000 7820 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1b6f87ff43404fec9ea1755cafe56b55 [sys.catalog]: This master's current role is: FOLLOWER
05:19:15.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:15.759080 7811 sys_catalog.cc:564] T 00000000000000000000000000000000 P 1b6f87ff43404fec9ea1755cafe56b55 [sys.catalog]: configured and running, proceeding with master startup.
05:19:15.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:15.794802 7763 tablet.cc:2369] T 00000000000000000000000000000000 P 1b6f87ff43404fec9ea1755cafe56b55: Can't schedule compaction. Clean time has not been advanced past its initial value.
05:19:15.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:15.795603 7831 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1b6f87ff43404fec9ea1755cafe56b55: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:19:15.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:15.795884 7831 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:19:15.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:15.784365 7733 master_runner.cc:186] Error getting master registration for 127.7.102.62:43223: OK, instance_id { permanent_uuid: "46ab27132ad44da4937b52e36a4f462e" instance_seqno: 1790572751954215 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
05:19:15.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:15.820472 7733 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:19:15.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:15.910173 7820 raft_consensus.cc:482] T 00000000000000000000000000000000 P 1b6f87ff43404fec9ea1755cafe56b55 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
05:19:15.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:15.910758 7820 raft_consensus.cc:504] T 00000000000000000000000000000000 P 1b6f87ff43404fec9ea1755cafe56b55 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "46ab27132ad44da4937b52e36a4f462e" member_type: VOTER last_known_addr { host: "127.7.102.62" port: 43223 } } peers { permanent_uuid: "5645663ae6034d8d9f14e74dfb612ec3" member_type: VOTER last_known_addr { host: "127.7.102.61" port: 35105 } } peers { permanent_uuid: "1b6f87ff43404fec9ea1755cafe56b55" member_type: VOTER last_known_addr { host: "127.7.102.60" port: 35175 } }
05:19:15.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:15.915189 7820 leader_election.cc:288] T 00000000000000000000000000000000 P 1b6f87ff43404fec9ea1755cafe56b55 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 46ab27132ad44da4937b52e36a4f462e (127.7.102.62:43223), 5645663ae6034d8d9f14e74dfb612ec3 (127.7.102.61:35105)
05:19:16.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.145795 7731 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.102.60" port: 35175 } attempt: 7
05:19:16.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.180661 7731 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5645663ae6034d8d9f14e74dfb612ec3: Bootstrap starting.
05:19:16.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.188134 7731 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 5645663ae6034d8d9f14e74dfb612ec3: Neither blocks nor log segments found. Creating new log.
05:19:16.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.190899 7731 log.cc:824] T 00000000000000000000000000000000 P 5645663ae6034d8d9f14e74dfb612ec3: Log is configured to *not* fsync() on all Append() calls
05:19:16.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.196843 7731 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5645663ae6034d8d9f14e74dfb612ec3: No bootstrap required, opened a new log
05:19:16.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.233474 7731 raft_consensus.cc:348] T 00000000000000000000000000000000 P 5645663ae6034d8d9f14e74dfb612ec3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "46ab27132ad44da4937b52e36a4f462e" member_type: VOTER last_known_addr { host: "127.7.102.62" port: 43223 } } peers { permanent_uuid: "5645663ae6034d8d9f14e74dfb612ec3" member_type: VOTER last_known_addr { host: "127.7.102.61" port: 35105 } } peers { permanent_uuid: "1b6f87ff43404fec9ea1755cafe56b55" member_type: VOTER last_known_addr { host: "127.7.102.60" port: 35175 } }
05:19:16.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.235321 7731 raft_consensus.cc:374] T 00000000000000000000000000000000 P 5645663ae6034d8d9f14e74dfb612ec3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
05:19:16.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.235637 7731 raft_consensus.cc:729] T 00000000000000000000000000000000 P 5645663ae6034d8d9f14e74dfb612ec3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5645663ae6034d8d9f14e74dfb612ec3, State: Initialized, Role: FOLLOWER
05:19:16.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.236526 7731 consensus_queue.cc:260] T 00000000000000000000000000000000 P 5645663ae6034d8d9f14e74dfb612ec3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "46ab27132ad44da4937b52e36a4f462e" member_type: VOTER last_known_addr { host: "127.7.102.62" port: 43223 } } peers { permanent_uuid: "5645663ae6034d8d9f14e74dfb612ec3" member_type: VOTER last_known_addr { host: "127.7.102.61" port: 35105 } } peers { permanent_uuid: "1b6f87ff43404fec9ea1755cafe56b55" member_type: VOTER last_known_addr { host: "127.7.102.60" port: 35175 } }
05:19:16.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.239274 7835 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5645663ae6034d8d9f14e74dfb612ec3 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "46ab27132ad44da4937b52e36a4f462e" member_type: VOTER last_known_addr { host: "127.7.102.62" port: 43223 } } peers { permanent_uuid: "5645663ae6034d8d9f14e74dfb612ec3" member_type: VOTER last_known_addr { host: "127.7.102.61" port: 35105 } } peers { permanent_uuid: "1b6f87ff43404fec9ea1755cafe56b55" member_type: VOTER last_known_addr { host: "127.7.102.60" port: 35175 } } }
05:19:16.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.240459 7835 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5645663ae6034d8d9f14e74dfb612ec3 [sys.catalog]: This master's current role is: FOLLOWER
05:19:16.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.241459 7731 sys_catalog.cc:564] T 00000000000000000000000000000000 P 5645663ae6034d8d9f14e74dfb612ec3 [sys.catalog]: configured and running, proceeding with master startup.
05:19:16.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.241665 7706 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1b6f87ff43404fec9ea1755cafe56b55" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5645663ae6034d8d9f14e74dfb612ec3" is_pre_election: true
05:19:16.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.242587 7706 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 5645663ae6034d8d9f14e74dfb612ec3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1b6f87ff43404fec9ea1755cafe56b55 in term 0.
05:19:16.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.244128 7761 leader_election.cc:302] T 00000000000000000000000000000000 P 1b6f87ff43404fec9ea1755cafe56b55 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1b6f87ff43404fec9ea1755cafe56b55, 5645663ae6034d8d9f14e74dfb612ec3; no voters:
05:19:16.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.245361 7820 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 1b6f87ff43404fec9ea1755cafe56b55 [term 0 FOLLOWER]: Leader pre-election won for term 1
05:19:16.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.245687 7820 raft_consensus.cc:482] T 00000000000000000000000000000000 P 1b6f87ff43404fec9ea1755cafe56b55 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
05:19:16.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.246069 7820 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 1b6f87ff43404fec9ea1755cafe56b55 [term 0 FOLLOWER]: Advancing to term 1
05:19:16.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.253429 7820 raft_consensus.cc:504] T 00000000000000000000000000000000 P 1b6f87ff43404fec9ea1755cafe56b55 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "46ab27132ad44da4937b52e36a4f462e" member_type: VOTER last_known_addr { host: "127.7.102.62" port: 43223 } } peers { permanent_uuid: "5645663ae6034d8d9f14e74dfb612ec3" member_type: VOTER last_known_addr { host: "127.7.102.61" port: 35105 } } peers { permanent_uuid: "1b6f87ff43404fec9ea1755cafe56b55" member_type: VOTER last_known_addr { host: "127.7.102.60" port: 35175 } }
05:19:16.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.272182 7706 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1b6f87ff43404fec9ea1755cafe56b55" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5645663ae6034d8d9f14e74dfb612ec3"
05:19:16.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.273191 7706 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 5645663ae6034d8d9f14e74dfb612ec3 [term 0 FOLLOWER]: Advancing to term 1
05:19:16.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.273777 7820 leader_election.cc:288] T 00000000000000000000000000000000 P 1b6f87ff43404fec9ea1755cafe56b55 [CANDIDATE]: Term 1 election: Requested vote from peers 46ab27132ad44da4937b52e36a4f462e (127.7.102.62:43223), 5645663ae6034d8d9f14e74dfb612ec3 (127.7.102.61:35105)
05:19:16.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.281297 7706 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 5645663ae6034d8d9f14e74dfb612ec3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1b6f87ff43404fec9ea1755cafe56b55 in term 1.
05:19:16.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.282790 7761 leader_election.cc:302] T 00000000000000000000000000000000 P 1b6f87ff43404fec9ea1755cafe56b55 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1b6f87ff43404fec9ea1755cafe56b55, 5645663ae6034d8d9f14e74dfb612ec3; no voters:
05:19:16.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.283946 7820 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 1b6f87ff43404fec9ea1755cafe56b55 [term 1 FOLLOWER]: Leader election won for term 1
05:19:16.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.287031 7820 raft_consensus.cc:686] T 00000000000000000000000000000000 P 1b6f87ff43404fec9ea1755cafe56b55 [term 1 LEADER]: Becoming Leader. State: Replica: 1b6f87ff43404fec9ea1755cafe56b55, State: Running, Role: LEADER
05:19:16.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.289328 7820 consensus_queue.cc:237] T 00000000000000000000000000000000 P 1b6f87ff43404fec9ea1755cafe56b55 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "46ab27132ad44da4937b52e36a4f462e" member_type: VOTER last_known_addr { host: "127.7.102.62" port: 43223 } } peers { permanent_uuid: "5645663ae6034d8d9f14e74dfb612ec3" member_type: VOTER last_known_addr { host: "127.7.102.61" port: 35105 } } peers { permanent_uuid: "1b6f87ff43404fec9ea1755cafe56b55" member_type: VOTER last_known_addr { host: "127.7.102.60" port: 35175 } }
05:19:16.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.312114 7838 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1b6f87ff43404fec9ea1755cafe56b55 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1b6f87ff43404fec9ea1755cafe56b55. Latest consensus state: current_term: 1 leader_uuid: "1b6f87ff43404fec9ea1755cafe56b55" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "46ab27132ad44da4937b52e36a4f462e" member_type: VOTER last_known_addr { host: "127.7.102.62" port: 43223 } } peers { permanent_uuid: "5645663ae6034d8d9f14e74dfb612ec3" member_type: VOTER last_known_addr { host: "127.7.102.61" port: 35105 } } peers { permanent_uuid: "1b6f87ff43404fec9ea1755cafe56b55" member_type: VOTER last_known_addr { host: "127.7.102.60" port: 35175 } } }
05:19:16.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.312994 7838 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1b6f87ff43404fec9ea1755cafe56b55 [sys.catalog]: This master's current role is: LEADER
05:19:16.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.317080 7844 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
05:19:16.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.329806 7844 catalog_manager.cc:1484] Initializing Kudu cluster ID...
05:19:16.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:16.350100 7849 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5645663ae6034d8d9f14e74dfb612ec3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:19:16.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:16.350802 7849 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:19:16.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.386624 7706 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 5645663ae6034d8d9f14e74dfb612ec3 [term 1 FOLLOWER]: Refusing update from remote peer 1b6f87ff43404fec9ea1755cafe56b55: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
05:19:16.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.388422 7838 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 1b6f87ff43404fec9ea1755cafe56b55 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5645663ae6034d8d9f14e74dfb612ec3" member_type: VOTER last_known_addr { host: "127.7.102.61" port: 35105 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
05:19:16.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.347782 7658 master_runner.cc:186] Error getting master registration for 127.7.102.62:43223: OK, instance_id { permanent_uuid: "46ab27132ad44da4937b52e36a4f462e" instance_seqno: 1790572751954215 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
05:19:16.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.396580 7658 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:19:16.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.423089 7835 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5645663ae6034d8d9f14e74dfb612ec3 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1b6f87ff43404fec9ea1755cafe56b55. Latest consensus state: current_term: 1 leader_uuid: "1b6f87ff43404fec9ea1755cafe56b55" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "46ab27132ad44da4937b52e36a4f462e" member_type: VOTER last_known_addr { host: "127.7.102.62" port: 43223 } } peers { permanent_uuid: "5645663ae6034d8d9f14e74dfb612ec3" member_type: VOTER last_known_addr { host: "127.7.102.61" port: 35105 } } peers { permanent_uuid: "1b6f87ff43404fec9ea1755cafe56b55" member_type: VOTER last_known_addr { host: "127.7.102.60" port: 35175 } } }
05:19:16.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.427836 7835 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5645663ae6034d8d9f14e74dfb612ec3 [sys.catalog]: This master's current role is: FOLLOWER
05:19:16.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.429486 7851 mvcc.cc:205] Tried to move back new op lower bound from 7334186010096902144 to 7334186009773154304. Current Snapshot: MvccSnapshot[applied={T|T < 7334186010096902144}]
05:19:16.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.433452 7820 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1b6f87ff43404fec9ea1755cafe56b55 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "1b6f87ff43404fec9ea1755cafe56b55" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "46ab27132ad44da4937b52e36a4f462e" member_type: VOTER last_known_addr { host: "127.7.102.62" port: 43223 } } peers { permanent_uuid: "5645663ae6034d8d9f14e74dfb612ec3" member_type: VOTER last_known_addr { host: "127.7.102.61" port: 35105 } } peers { permanent_uuid: "1b6f87ff43404fec9ea1755cafe56b55" member_type: VOTER last_known_addr { host: "127.7.102.60" port: 35175 } } }
05:19:16.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.434180 7820 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1b6f87ff43404fec9ea1755cafe56b55 [sys.catalog]: This master's current role is: LEADER
05:19:16.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.442247 7844 catalog_manager.cc:1347] Generated new cluster ID: 6c317f34464640ffb7f8cb3da776412e
05:19:16.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.442556 7844 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
05:19:16.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.452710 7835 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5645663ae6034d8d9f14e74dfb612ec3 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "1b6f87ff43404fec9ea1755cafe56b55" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "46ab27132ad44da4937b52e36a4f462e" member_type: VOTER last_known_addr { host: "127.7.102.62" port: 43223 } } peers { permanent_uuid: "5645663ae6034d8d9f14e74dfb612ec3" member_type: VOTER last_known_addr { host: "127.7.102.61" port: 35105 } } peers { permanent_uuid: "1b6f87ff43404fec9ea1755cafe56b55" member_type: VOTER last_known_addr { host: "127.7.102.60" port: 35175 } } }
05:19:16.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.453444 7835 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5645663ae6034d8d9f14e74dfb612ec3 [sys.catalog]: This master's current role is: FOLLOWER
05:19:16.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.488845 7844 catalog_manager.cc:1370] Generated new certificate authority record
05:19:16.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.496064 7844 catalog_manager.cc:1504] Loading token signing keys...
05:19:16.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.531549 7844 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 1b6f87ff43404fec9ea1755cafe56b55: Generated new TSK 0
05:19:16.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.534015 7844 catalog_manager.cc:1514] Initializing in-progress tserver states...
05:19:16.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.575816 7576 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskHSJIN2/build/tsan/bin/kudu
05:19:16.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskHSJIN2/build/tsan/bin/kudu
05:19:16.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/ts-0/wal
05:19:16.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/ts-0/data
05:19:16.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:19:16.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:19:16.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:19:16.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:19:16.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:19:16.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:19:16.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/ts-0/logs
05:19:16.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/ts-0/data/info.pb
05:19:16.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:19:16.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:19:16.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:19:16.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:19:16.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:19:16.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:19:16.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:19:16.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:19:16.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
05:19:16.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:19:16.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.102.1:0
05:19:16.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.7.102.1
05:19:16.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.102.1
05:19:16.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:19:16.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.7.102.62:43223,127.7.102.61:35105,127.7.102.60:35175
05:19:16.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.102.20:44503
05:19:16.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:19:16.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:19:16.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
05:19:16.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.614548 7656 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.102.60" port: 35175 } attempt: 7
05:19:16.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.700246 7656 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 46ab27132ad44da4937b52e36a4f462e: Bootstrap starting.
05:19:16.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.719656 7656 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 46ab27132ad44da4937b52e36a4f462e: Neither blocks nor log segments found. Creating new log.
05:19:16.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.722214 7656 log.cc:824] T 00000000000000000000000000000000 P 46ab27132ad44da4937b52e36a4f462e: Log is configured to *not* fsync() on all Append() calls
05:19:16.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.747226 7656 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 46ab27132ad44da4937b52e36a4f462e: No bootstrap required, opened a new log
05:19:16.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.804723 7656 raft_consensus.cc:348] T 00000000000000000000000000000000 P 46ab27132ad44da4937b52e36a4f462e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "46ab27132ad44da4937b52e36a4f462e" member_type: VOTER last_known_addr { host: "127.7.102.62" port: 43223 } } peers { permanent_uuid: "5645663ae6034d8d9f14e74dfb612ec3" member_type: VOTER last_known_addr { host: "127.7.102.61" port: 35105 } } peers { permanent_uuid: "1b6f87ff43404fec9ea1755cafe56b55" member_type: VOTER last_known_addr { host: "127.7.102.60" port: 35175 } }
05:19:16.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.815308 7656 raft_consensus.cc:374] T 00000000000000000000000000000000 P 46ab27132ad44da4937b52e36a4f462e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
05:19:16.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.815997 7656 raft_consensus.cc:729] T 00000000000000000000000000000000 P 46ab27132ad44da4937b52e36a4f462e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 46ab27132ad44da4937b52e36a4f462e, State: Initialized, Role: FOLLOWER
05:19:16.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.817232 7656 consensus_queue.cc:260] T 00000000000000000000000000000000 P 46ab27132ad44da4937b52e36a4f462e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "46ab27132ad44da4937b52e36a4f462e" member_type: VOTER last_known_addr { host: "127.7.102.62" port: 43223 } } peers { permanent_uuid: "5645663ae6034d8d9f14e74dfb612ec3" member_type: VOTER last_known_addr { host: "127.7.102.61" port: 35105 } } peers { permanent_uuid: "1b6f87ff43404fec9ea1755cafe56b55" member_type: VOTER last_known_addr { host: "127.7.102.60" port: 35175 } }
05:19:16.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.830147 7656 sys_catalog.cc:564] T 00000000000000000000000000000000 P 46ab27132ad44da4937b52e36a4f462e [sys.catalog]: configured and running, proceeding with master startup.
05:19:16.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.830752 7629 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 46ab27132ad44da4937b52e36a4f462e [term 0 FOLLOWER]: Advancing to term 1
05:19:16.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.831944 7630 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1b6f87ff43404fec9ea1755cafe56b55" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "46ab27132ad44da4937b52e36a4f462e"
05:19:16.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.831246 7631 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1b6f87ff43404fec9ea1755cafe56b55" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "46ab27132ad44da4937b52e36a4f462e" is_pre_election: true
05:19:16.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.838493 7629 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 46ab27132ad44da4937b52e36a4f462e [term 1 FOLLOWER]: Refusing update from remote peer 1b6f87ff43404fec9ea1755cafe56b55: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
05:19:16.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.840580 7820 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 1b6f87ff43404fec9ea1755cafe56b55 [LEADER]: Connected to new peer: Peer: permanent_uuid: "46ab27132ad44da4937b52e36a4f462e" member_type: VOTER last_known_addr { host: "127.7.102.62" port: 43223 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
05:19:16.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.857436 7859 sys_catalog.cc:455] T 00000000000000000000000000000000 P 46ab27132ad44da4937b52e36a4f462e [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "46ab27132ad44da4937b52e36a4f462e" member_type: VOTER last_known_addr { host: "127.7.102.62" port: 43223 } } peers { permanent_uuid: "5645663ae6034d8d9f14e74dfb612ec3" member_type: VOTER last_known_addr { host: "127.7.102.61" port: 35105 } } peers { permanent_uuid: "1b6f87ff43404fec9ea1755cafe56b55" member_type: VOTER last_known_addr { host: "127.7.102.60" port: 35175 } } }
05:19:16.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.860351 7859 sys_catalog.cc:458] T 00000000000000000000000000000000 P 46ab27132ad44da4937b52e36a4f462e [sys.catalog]: This master's current role is: FOLLOWER
05:19:16.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:16.909350 7610 tablet.cc:2369] T 00000000000000000000000000000000 P 46ab27132ad44da4937b52e36a4f462e: Can't schedule compaction. Clean time has not been advanced past its initial value.
05:19:16.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.936358 7859 sys_catalog.cc:455] T 00000000000000000000000000000000 P 46ab27132ad44da4937b52e36a4f462e [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "1b6f87ff43404fec9ea1755cafe56b55" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "46ab27132ad44da4937b52e36a4f462e" member_type: VOTER last_known_addr { host: "127.7.102.62" port: 43223 } } peers { permanent_uuid: "5645663ae6034d8d9f14e74dfb612ec3" member_type: VOTER last_known_addr { host: "127.7.102.61" port: 35105 } } peers { permanent_uuid: "1b6f87ff43404fec9ea1755cafe56b55" member_type: VOTER last_known_addr { host: "127.7.102.60" port: 35175 } } }
05:19:16.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.937098 7859 sys_catalog.cc:458] T 00000000000000000000000000000000 P 46ab27132ad44da4937b52e36a4f462e [sys.catalog]: This master's current role is: FOLLOWER
05:19:16.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.938194 7861 sys_catalog.cc:455] T 00000000000000000000000000000000 P 46ab27132ad44da4937b52e36a4f462e [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1b6f87ff43404fec9ea1755cafe56b55. Latest consensus state: current_term: 1 leader_uuid: "1b6f87ff43404fec9ea1755cafe56b55" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "46ab27132ad44da4937b52e36a4f462e" member_type: VOTER last_known_addr { host: "127.7.102.62" port: 43223 } } peers { permanent_uuid: "5645663ae6034d8d9f14e74dfb612ec3" member_type: VOTER last_known_addr { host: "127.7.102.61" port: 35105 } } peers { permanent_uuid: "1b6f87ff43404fec9ea1755cafe56b55" member_type: VOTER last_known_addr { host: "127.7.102.60" port: 35175 } } }
05:19:16.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:16.938953 7861 sys_catalog.cc:458] T 00000000000000000000000000000000 P 46ab27132ad44da4937b52e36a4f462e [sys.catalog]: This master's current role is: FOLLOWER
05:19:17.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:17.008554 7875 catalog_manager.cc:1259] Loaded cluster ID: 6c317f34464640ffb7f8cb3da776412e
05:19:17.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:17.009183 7875 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 46ab27132ad44da4937b52e36a4f462e: loading cluster ID for follower catalog manager: success
05:19:17.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:17.030761 7875 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 46ab27132ad44da4937b52e36a4f462e: acquiring CA information for follower catalog manager: success
05:19:17.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:17.048393 7875 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 46ab27132ad44da4937b52e36a4f462e: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
05:19:17.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:17.182210 7856 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:19:17.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:17.183254 7856 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:19:17.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:17.184296 7856 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:19:17.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:17.245151 7856 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:19:17.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:17.246951 7856 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.102.1
05:19:17.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:17.313241 7856 tablet_server_runner.cc:78] Tablet server non-default flags:
05:19:17.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:19:17.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.102.20:44503
05:19:17.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:19:17.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:19:17.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/ts-0/data
05:19:17.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/ts-0/wal
05:19:17.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:19:17.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:19:17.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.102.1:0
05:19:17.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:19:17.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:19:17.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:19:17.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/ts-0/data/info.pb
05:19:17.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.102.1
05:19:17.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:19:17.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.7.102.62:43223,127.7.102.61:35105,127.7.102.60:35175
05:19:17.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:19:17.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:19:17.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:19:17.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:19:17.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:19:17.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.7.102.1
05:19:17.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/ts-0/logs
05:19:17.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:19:17.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:19:17.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:19:17.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
05:19:17.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:19:17.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:19:17.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:19:17.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 04:35:01 UTC on ccycloud.quasar-alllgi.root.comops.site
05:19:17.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676563
05:19:17.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:19:17.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:17.317782 7856 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:19:17.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:17.320509 7856 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:19:17.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:17.358491 7849 catalog_manager.cc:1259] Loaded cluster ID: 6c317f34464640ffb7f8cb3da776412e
05:19:17.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:17.360750 7849 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 5645663ae6034d8d9f14e74dfb612ec3: loading cluster ID for follower catalog manager: success
05:19:17.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:17.376178 7883 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:19:17.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:17.392999 7849 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 5645663ae6034d8d9f14e74dfb612ec3: acquiring CA information for follower catalog manager: success
05:19:17.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:17.409878 7849 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 5645663ae6034d8d9f14e74dfb612ec3: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
05:19:17.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:17.455554 7820 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1b6f87ff43404fec9ea1755cafe56b55 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "1b6f87ff43404fec9ea1755cafe56b55" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "46ab27132ad44da4937b52e36a4f462e" member_type: VOTER last_known_addr { host: "127.7.102.62" port: 43223 } } peers { permanent_uuid: "5645663ae6034d8d9f14e74dfb612ec3" member_type: VOTER last_known_addr { host: "127.7.102.61" port: 35105 } } peers { permanent_uuid: "1b6f87ff43404fec9ea1755cafe56b55" member_type: VOTER last_known_addr { host: "127.7.102.60" port: 35175 } } }
05:19:17.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:17.456566 7820 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1b6f87ff43404fec9ea1755cafe56b55 [sys.catalog]: This master's current role is: LEADER
05:19:20.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:17.377986 7882 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:19:20.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:18.746527 7881 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 7856
05:19:20.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:20.174172 7881 kernel_stack_watchdog.cc:198] Thread 7856 stuck at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:637 for 367ms:
05:19:20.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
05:19:20.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
05:19:20.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
05:19:20.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
05:19:20.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:20.176748 7886 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:19:20.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:20.177650 7884 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
05:19:20.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:20.179445 7856 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 2.803s user 0.824s sys 1.409s
05:19:20.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:20.179770 7856 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 2.803s user 0.824s sys 1.409s
05:19:20.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:20.180011 7856 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
05:19:20.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:20.181434 7856 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:19:20.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:20.184192 7856 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:19:20.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:20.185662 7856 hybrid_clock.cc:648] HybridClock initialized: now 1790572760185601 us; error 69 us; skew 500 ppm
05:19:20.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:20.187292 7856 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:19:20.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:20.196600 7856 webserver.cc:460] Webserver started at http://127.7.102.1:36859/ using document root <none> and password file <none>
05:19:20.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:20.198010 7856 fs_manager.cc:359] Metadata directory not provided
05:19:20.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:20.198331 7856 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:19:20.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:20.198973 7856 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:19:20.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:20.206619 7856 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/ts-0/data/instance:
05:19:20.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dc16e53e3da94eb4b8af67c139ae7ed0"
05:19:20.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 05:19:20 on dist-test-slave-lhv2"
05:19:20.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:20.208178 7856 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/ts-0/wal/instance:
05:19:20.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dc16e53e3da94eb4b8af67c139ae7ed0"
05:19:20.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 05:19:20 on dist-test-slave-lhv2"
05:19:20.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:20.218279 7856 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.009s sys 0.001s
05:19:20.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:20.225787 7896 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:19:20.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:20.227223 7856 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.004s
05:19:20.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:20.227628 7856 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/ts-0/data,/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/ts-0/wal
05:19:20.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dc16e53e3da94eb4b8af67c139ae7ed0"
05:19:20.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 05:19:20 on dist-test-slave-lhv2"
05:19:20.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:20.228063 7856 fs_report.cc:383] FS layout report
05:19:20.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:19:20.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/ts-0/wal
05:19:20.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/ts-0/wal
05:19:20.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/ts-0/data/data
05:19:20.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:19:20.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:19:20.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:19:20.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:19:20.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:19:20.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:19:20.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:19:20.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:19:20.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:19:20.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:19:20.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:19:20.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:20.282662 7856 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:19:20.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:20.284403 7856 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:19:20.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:20.284829 7856 kserver.cc:163] Server-wide thread pool size limit: 3276
05:19:20.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:20.288359 7856 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
05:19:20.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:20.293443 7856 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
05:19:20.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:20.293645 7856 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
05:19:20.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:20.293840 7856 ts_tablet_manager.cc:610] Registered 0 tablets
05:19:20.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:20.293974 7856 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
05:19:20.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:20.541255 7856 rpc_server.cc:307] RPC server started. Bound to: 127.7.102.1:39465
05:19:20.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:20.541566 8009 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.102.1:39465 every 8 connection(s)
05:19:20.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:20.544199 7856 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/ts-0/data/info.pb
05:19:20.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:20.548930 7576 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskHSJIN2/build/tsan/bin/kudu as pid 7856
05:19:20.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:20.549778 7576 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/ts-0/wal/instance
05:19:20.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:20.581022 7576 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskHSJIN2/build/tsan/bin/kudu
05:19:20.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskHSJIN2/build/tsan/bin/kudu
05:19:20.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/ts-1/wal
05:19:20.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/ts-1/data
05:19:20.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:19:20.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:19:20.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:19:20.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:19:20.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:19:20.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:19:20.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/ts-1/logs
05:19:20.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/ts-1/data/info.pb
05:19:20.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:19:20.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:19:20.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:19:20.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:19:20.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:19:20.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:19:20.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:19:20.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:19:20.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
05:19:20.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:19:20.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.102.2:0
05:19:20.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.7.102.2
05:19:20.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.102.2
05:19:20.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:19:20.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.7.102.62:43223,127.7.102.61:35105,127.7.102.60:35175
05:19:20.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.102.20:44503
05:19:20.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:19:20.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:19:20.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
05:19:20.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:20.652348 8010 heartbeater.cc:347] Connected to a master server at 127.7.102.60:35175
05:19:20.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:20.652978 8010 heartbeater.cc:464] Registering TS with master...
05:19:20.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:20.654493 8010 heartbeater.cc:511] Master 127.7.102.60:35175 requested a full tablet report, sending...
05:19:20.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:20.659245 7774 ts_manager.cc:194] Registered new tserver with Master: dc16e53e3da94eb4b8af67c139ae7ed0 (127.7.102.1:39465)
05:19:20.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:20.666877 7774 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.102.1:38299
05:19:20.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:20.731353 8016 heartbeater.cc:347] Connected to a master server at 127.7.102.62:43223
05:19:20.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:20.731762 8016 heartbeater.cc:464] Registering TS with master...
05:19:20.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:20.732549 8016 heartbeater.cc:511] Master 127.7.102.62:43223 requested a full tablet report, sending...
05:19:20.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:20.736802 7621 ts_manager.cc:194] Registered new tserver with Master: dc16e53e3da94eb4b8af67c139ae7ed0 (127.7.102.1:39465)
05:19:20.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:20.749500 8011 heartbeater.cc:347] Connected to a master server at 127.7.102.61:35105
05:19:20.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:20.749910 8011 heartbeater.cc:464] Registering TS with master...
05:19:20.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:20.750654 8011 heartbeater.cc:511] Master 127.7.102.61:35105 requested a full tablet report, sending...
05:19:20.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:20.753970 7696 ts_manager.cc:194] Registered new tserver with Master: dc16e53e3da94eb4b8af67c139ae7ed0 (127.7.102.1:39465)
05:19:21.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:21.103055 8018 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:19:21.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:21.103602 8018 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:19:21.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:21.104099 8018 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:19:21.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:21.136672 8018 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:19:21.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:21.137468 8018 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.102.2
05:19:21.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:21.173393 8018 tablet_server_runner.cc:78] Tablet server non-default flags:
05:19:21.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:19:21.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.102.20:44503
05:19:21.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:19:21.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:19:21.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/ts-1/data
05:19:21.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/ts-1/wal
05:19:21.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:19:21.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:19:21.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.102.2:0
05:19:21.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:19:21.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:19:21.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:19:21.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/ts-1/data/info.pb
05:19:21.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.102.2
05:19:21.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:19:21.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.7.102.62:43223,127.7.102.61:35105,127.7.102.60:35175
05:19:21.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:19:21.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:19:21.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:19:21.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:19:21.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:19:21.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.7.102.2
05:19:21.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/ts-1/logs
05:19:21.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:19:21.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:19:21.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:19:21.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
05:19:21.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:19:21.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:19:21.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:19:21.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 04:35:01 UTC on ccycloud.quasar-alllgi.root.comops.site
05:19:21.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676563
05:19:21.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:19:21.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:21.174665 8018 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:19:21.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:21.176805 8018 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:19:21.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:21.201960 8027 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:19:21.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:21.674243 8010 heartbeater.cc:503] Master 127.7.102.60:35175 was elected leader, sending a full tablet report...
05:19:22.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:21.202546 8028 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:19:22.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:21.204630 8018 server_base.cc:1034] running on GCE node
05:19:22.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:21.202590 8030 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:19:22.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:22.425307 8018 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:19:22.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:22.427650 8018 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:19:22.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:22.429033 8018 hybrid_clock.cc:648] HybridClock initialized: now 1790572762428996 us; error 48 us; skew 500 ppm
05:19:22.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:22.429975 8018 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:19:22.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:22.436146 8018 webserver.cc:460] Webserver started at http://127.7.102.2:46221/ using document root <none> and password file <none>
05:19:22.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:22.436988 8018 fs_manager.cc:359] Metadata directory not provided
05:19:22.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:22.437170 8018 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:19:22.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:22.437529 8018 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:19:22.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:22.442543 8018 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/ts-1/data/instance:
05:19:22.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2e16b5a1a1a245bcbe57bda91eb92701"
05:19:22.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 05:19:22 on dist-test-slave-lhv2"
05:19:22.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:22.443768 8018 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/ts-1/wal/instance:
05:19:22.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2e16b5a1a1a245bcbe57bda91eb92701"
05:19:22.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 05:19:22 on dist-test-slave-lhv2"
05:19:22.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:22.450964 8018 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
05:19:22.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:22.456645 8039 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:19:22.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:22.457759 8018 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.003s
05:19:22.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:22.458132 8018 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/ts-1/data,/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/ts-1/wal
05:19:22.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2e16b5a1a1a245bcbe57bda91eb92701"
05:19:22.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 05:19:22 on dist-test-slave-lhv2"
05:19:22.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:22.458527 8018 fs_report.cc:383] FS layout report
05:19:22.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:19:22.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/ts-1/wal
05:19:22.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/ts-1/wal
05:19:22.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/ts-1/data/data
05:19:22.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:19:22.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:19:22.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:19:22.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:19:22.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:19:22.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:19:22.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:19:22.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:19:22.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:19:22.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:19:22.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:19:22.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:22.498329 8018 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:19:22.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:22.499933 8018 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:19:22.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:22.500435 8018 kserver.cc:163] Server-wide thread pool size limit: 3276
05:19:22.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:22.502974 8018 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
05:19:22.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:22.507128 8018 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
05:19:22.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:22.507417 8018 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.001s
05:19:22.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:22.507747 8018 ts_tablet_manager.cc:610] Registered 0 tablets
05:19:22.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:22.507942 8018 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
05:19:22.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:22.666369 8018 rpc_server.cc:307] RPC server started. Bound to: 127.7.102.2:39641
05:19:22.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:22.666519 8151 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.102.2:39641 every 8 connection(s)
05:19:22.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:22.669286 8018 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/ts-1/data/info.pb
05:19:22.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:22.673051 7576 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskHSJIN2/build/tsan/bin/kudu as pid 8018
05:19:22.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:22.673506 7576 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/ts-1/wal/instance
05:19:22.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:22.690747 7576 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskHSJIN2/build/tsan/bin/kudu
05:19:22.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskHSJIN2/build/tsan/bin/kudu
05:19:22.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/ts-2/wal
05:19:22.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/ts-2/data
05:19:22.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:19:22.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:19:22.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:19:22.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:19:22.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:19:22.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:19:22.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/ts-2/logs
05:19:22.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/ts-2/data/info.pb
05:19:22.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:19:22.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:19:22.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:19:22.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:19:22.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:19:22.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:19:22.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:19:22.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:19:22.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
05:19:22.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:19:22.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.102.3:0
05:19:22.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.7.102.3
05:19:22.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.102.3
05:19:22.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:19:22.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.7.102.62:43223,127.7.102.61:35105,127.7.102.60:35175
05:19:22.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.102.20:44503
05:19:22.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:19:22.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:19:22.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
05:19:22.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:22.745507 8152 heartbeater.cc:347] Connected to a master server at 127.7.102.60:35175
05:19:22.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:22.745827 8154 heartbeater.cc:347] Connected to a master server at 127.7.102.62:43223
05:19:22.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:22.745991 8152 heartbeater.cc:464] Registering TS with master...
05:19:22.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:22.746213 8154 heartbeater.cc:464] Registering TS with master...
05:19:22.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:22.746768 8153 heartbeater.cc:347] Connected to a master server at 127.7.102.61:35105
05:19:22.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:22.747210 8153 heartbeater.cc:464] Registering TS with master...
05:19:22.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:22.747398 8152 heartbeater.cc:511] Master 127.7.102.60:35175 requested a full tablet report, sending...
05:19:22.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:22.747660 8154 heartbeater.cc:511] Master 127.7.102.62:43223 requested a full tablet report, sending...
05:19:22.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:22.748018 8153 heartbeater.cc:511] Master 127.7.102.61:35105 requested a full tablet report, sending...
05:19:22.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:22.750492 7773 ts_manager.cc:194] Registered new tserver with Master: 2e16b5a1a1a245bcbe57bda91eb92701 (127.7.102.2:39641)
05:19:22.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:22.750507 7621 ts_manager.cc:194] Registered new tserver with Master: 2e16b5a1a1a245bcbe57bda91eb92701 (127.7.102.2:39641)
05:19:22.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:22.750674 7696 ts_manager.cc:194] Registered new tserver with Master: 2e16b5a1a1a245bcbe57bda91eb92701 (127.7.102.2:39641)
05:19:22.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:22.752599 7773 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.102.2:38683
05:19:23.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:23.066007 8162 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:19:23.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:23.066589 8162 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:19:23.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:23.067278 8162 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:19:23.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:23.098927 8162 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:19:23.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:23.099792 8162 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.102.3
05:19:23.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:23.134548 8162 tablet_server_runner.cc:78] Tablet server non-default flags:
05:19:23.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:19:23.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.102.20:44503
05:19:23.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:19:23.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:19:23.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/ts-2/data
05:19:23.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/ts-2/wal
05:19:23.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:19:23.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:19:23.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.102.3:0
05:19:23.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:19:23.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:19:23.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:19:23.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/ts-2/data/info.pb
05:19:23.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.102.3
05:19:23.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:19:23.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.7.102.62:43223,127.7.102.61:35105,127.7.102.60:35175
05:19:23.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:19:23.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:19:23.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:19:23.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:19:23.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:19:23.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.7.102.3
05:19:23.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/ts-2/logs
05:19:23.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:19:23.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:19:23.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:19:23.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
05:19:23.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:19:23.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:19:23.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:19:23.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 04:35:01 UTC on ccycloud.quasar-alllgi.root.comops.site
05:19:23.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676563
05:19:23.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:19:23.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:23.135823 8162 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:19:23.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:23.137562 8162 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:19:23.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:23.158903 8169 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:19:23.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:23.756672 8152 heartbeater.cc:503] Master 127.7.102.60:35175 was elected leader, sending a full tablet report...
05:19:24.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:23.159426 8168 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:19:24.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:23.161863 8162 server_base.cc:1034] running on GCE node
05:19:24.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:23.158926 8171 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:19:24.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:24.390153 8162 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:19:24.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:24.392998 8162 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:19:24.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:24.394423 8162 hybrid_clock.cc:648] HybridClock initialized: now 1790572764394380 us; error 60 us; skew 500 ppm
05:19:24.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:24.395254 8162 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:19:24.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:24.408017 8162 webserver.cc:460] Webserver started at http://127.7.102.3:42035/ using document root <none> and password file <none>
05:19:24.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:24.409036 8162 fs_manager.cc:359] Metadata directory not provided
05:19:24.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:24.409224 8162 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:19:24.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:24.409591 8162 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:19:24.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:24.414705 8162 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/ts-2/data/instance:
05:19:24.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d67cac3ce30341cf96d40d9941468586"
05:19:24.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 05:19:24 on dist-test-slave-lhv2"
05:19:24.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:24.415869 8162 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/ts-2/wal/instance:
05:19:24.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d67cac3ce30341cf96d40d9941468586"
05:19:24.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 05:19:24 on dist-test-slave-lhv2"
05:19:24.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:24.423307 8162 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
05:19:24.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:24.428838 8179 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:19:24.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:24.430233 8162 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.003s
05:19:24.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:24.430555 8162 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/ts-2/data,/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/ts-2/wal
05:19:24.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d67cac3ce30341cf96d40d9941468586"
05:19:24.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 05:19:24 on dist-test-slave-lhv2"
05:19:24.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:24.430883 8162 fs_report.cc:383] FS layout report
05:19:24.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:19:24.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/ts-2/wal
05:19:24.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/ts-2/wal
05:19:24.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/ts-2/data/data
05:19:24.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:19:24.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:19:24.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:19:24.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:19:24.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:19:24.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:19:24.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:19:24.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:19:24.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:19:24.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:19:24.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:19:24.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:24.467006 8162 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:19:24.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:24.468817 8162 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:19:24.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:24.469226 8162 kserver.cc:163] Server-wide thread pool size limit: 3276
05:19:24.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:24.471926 8162 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
05:19:24.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:24.476186 8162 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
05:19:24.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:24.476384 8162 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
05:19:24.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:24.476584 8162 ts_tablet_manager.cc:610] Registered 0 tablets
05:19:24.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:24.476712 8162 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
05:19:24.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:24.631639 8162 rpc_server.cc:307] RPC server started. Bound to: 127.7.102.3:36049
05:19:24.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:24.631680 8291 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.102.3:36049 every 8 connection(s)
05:19:24.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:24.634351 8162 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/ts-2/data/info.pb
05:19:24.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:24.638324 7576 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskHSJIN2/build/tsan/bin/kudu as pid 8162
05:19:24.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:24.638890 7576 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1863532428969760244/ts-2/wal/instance
05:19:24.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:24.706461 8296 heartbeater.cc:347] Connected to a master server at 127.7.102.62:43223
05:19:24.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:24.706907 8296 heartbeater.cc:464] Registering TS with master...
05:19:24.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:24.708071 8296 heartbeater.cc:511] Master 127.7.102.62:43223 requested a full tablet report, sending...
05:19:24.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:24.709097 8293 heartbeater.cc:347] Connected to a master server at 127.7.102.61:35105
05:19:24.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:24.709380 8293 heartbeater.cc:464] Registering TS with master...
05:19:24.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:24.710139 8293 heartbeater.cc:511] Master 127.7.102.61:35105 requested a full tablet report, sending...
05:19:24.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:24.711207 7620 ts_manager.cc:194] Registered new tserver with Master: d67cac3ce30341cf96d40d9941468586 (127.7.102.3:36049)
05:19:24.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:24.713310 8292 heartbeater.cc:347] Connected to a master server at 127.7.102.60:35175
05:19:24.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:24.713558 8292 heartbeater.cc:464] Registering TS with master...
05:19:24.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:24.713822 7695 ts_manager.cc:194] Registered new tserver with Master: d67cac3ce30341cf96d40d9941468586 (127.7.102.3:36049)
05:19:24.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:24.714108 8292 heartbeater.cc:511] Master 127.7.102.60:35175 requested a full tablet report, sending...
05:19:24.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:24.716050 7773 ts_manager.cc:194] Registered new tserver with Master: d67cac3ce30341cf96d40d9941468586 (127.7.102.3:36049)
05:19:24.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:24.718181 7773 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.102.3:38101
05:19:24.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:24.734699 7576 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
05:19:24.746 [DEBUG - main] (MiniKuduCluster.java:195) Response:
05:19:24.747 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
05:19:24.758 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.7.102.62"
port: 43223
}
bound_http_address {
host: "127.7.102.62"
port: 41079
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.7.102.61"
port: 35105
}
bound_http_address {
host: "127.7.102.61"
port: 33213
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.7.102.60"
port: 35175
}
bound_http_address {
host: "127.7.102.60"
port: 34921
}
}
}
05:19:24.765 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
05:19:24.768 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.7.102.1"
port: 39465
}
bound_http_address {
host: "127.7.102.1"
port: 36859
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.7.102.2"
port: 39641
}
bound_http_address {
host: "127.7.102.2"
port: 46221
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.7.102.3"
port: 36049
}
bound_http_address {
host: "127.7.102.3"
port: 42035
}
}
}
05:19:24.768 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
05:19:24.959 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
05:19:25.083 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskHSJIN2/test-tmp
05:19:25.084 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
05:19:25.084 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskHSJIN2/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
05:19:25.094 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
05:19:25.427 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:19:25.428 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
05:19:25.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:25.433884 8303 mini_oidc.cc:152] Starting JWKS server
05:19:25.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:25.437633 8303 webserver.cc:460] Webserver started at http://127.0.0.1:37289/ using document root <none> and password file <none>
05:19:25.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:25.438051 8303 mini_oidc.cc:189] Starting OIDC Discovery server
05:19:25.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:25.440245 8303 webserver.cc:460] Webserver started at http://127.0.0.1:42727/ using document root <none> and password file <none>
05:19:25.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
05:19:25.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/krb5kdc/principal' for realm 'KRBTEST.COM',
05:19:25.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
05:19:25.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 05:19:25 dist-test-slave-lhv2 krb5kdc[8315](info): setting up network...
05:19:25.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
05:19:25.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 05:19:25 dist-test-slave-lhv2 krb5kdc[8315](info): set up 2 sockets
05:19:25.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 05:19:25 dist-test-slave-lhv2 krb5kdc[8315](info): commencing operation
05:19:25.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
05:19:25.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:25.721665 8292 heartbeater.cc:503] Master 127.7.102.60:35175 was elected leader, sending a full tablet report...
05:19:26.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:26.557966 8303 mini_kdc.cc:121] Time spent starting KDC: real 1.092s user 0.002s sys 0.019s
05:19:26.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
05:19:26.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
05:19:26.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
05:19:26.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
05:19:26.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
05:19:26.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
05:19:26.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
05:19:26.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
05:19:26.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
05:19:26.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
05:19:26.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/krb5kdc/test-user.keytab.
05:19:26.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/krb5kdc/test-user.keytab.
05:19:26.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 05:19:26 dist-test-slave-lhv2 krb5kdc[8315](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790572766, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:19:26.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
05:19:26.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-28T05:19:26Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
05:19:26.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-28T05:19:26Z Disabled control of system clock
05:19:26.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.8.27.254@KRBTEST.COM; defaulting to no policy
05:19:26.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
05:19:26.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.8.27.254@KRBTEST.COM" created.
05:19:26.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
05:19:26.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.8.27.254 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/krb5kdc/kudu_127.8.27.254.keytab.
05:19:26.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.8.27.254 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/krb5kdc/kudu_127.8.27.254.keytab.
05:19:26.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:26.770691 8303 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskHSJIN2/build/tsan/bin/kudu
05:19:26.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskHSJIN2/build/tsan/bin/kudu
05:19:26.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-0/wal
05:19:26.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-0/data
05:19:26.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:19:26.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:19:26.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:19:26.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:19:26.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:19:26.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:19:26.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-0/logs
05:19:26.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-0/data/info.pb
05:19:26.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:19:26.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:19:26.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:19:26.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:19:26.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:19:26.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:19:26.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:19:26.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:19:26.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:19:26.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:19:26.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:19:26.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:19:26.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.27.254:34253
05:19:26.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.27.254
05:19:26.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:19:26.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/krb5kdc/kudu_127.8.27.254.keytab
05:19:26.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.27.254
05:19:26.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:19:26.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
05:19:26.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
05:19:26.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.27.212:40337
05:19:26.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:19:26.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:19:26.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:19:26.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:19:26.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.27.254:34253,127.8.27.253:44545,127.8.27.252:39249
05:19:26.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:19:26.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37289/jwks.json
05:19:26.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
05:19:27.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:27.089871 8328 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:19:27.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:27.090428 8328 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:19:27.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:27.090989 8328 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:19:27.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:27.125985 8328 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
05:19:27.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:27.126217 8328 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:19:27.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:27.126485 8328 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:19:27.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:27.126675 8328 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:19:27.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:27.126857 8328 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:19:27.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:27.127041 8328 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:19:27.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:27.127352 8328 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37289/jwks.json
05:19:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:27.164585 8328 master_runner.cc:386] Master server non-default flags:
05:19:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:19:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.27.212:40337
05:19:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:19:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:19:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-0/data
05:19:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-0/wal
05:19:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
05:19:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:19:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.27.254:34253,127.8.27.253:44545,127.8.27.252:39249
05:19:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/krb5kdc/kudu_127.8.27.254.keytab
05:19:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.27.254
05:19:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:19:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:19:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:19:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.27.254:34253
05:19:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:19:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:19:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:19:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37289/jwks.json
05:19:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:19:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
05:19:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
05:19:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:19:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:19:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-0/data/info.pb
05:19:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.27.254
05:19:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:19:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:19:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:19:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:19:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:19:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:19:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-0/logs
05:19:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:19:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:19:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:19:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:19:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:19:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:19:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:19:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 04:35:01 UTC on ccycloud.quasar-alllgi.root.comops.site
05:19:27.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676563
05:19:27.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:19:27.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:27.165705 8328 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:19:27.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:27.166656 8328 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:19:27.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:27.184602 8335 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:19:28.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:27.184587 8338 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:19:28.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:27.184587 8336 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:19:28.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:28.535106 8337 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1352 milliseconds
05:19:28.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:28.535240 8328 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
05:19:28.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:28.539835 8328 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:19:28.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:28.543110 8328 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:19:28.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:28.544739 8328 hybrid_clock.cc:648] HybridClock initialized: now 1790572768544703 us; error 51 us; skew 500 ppm
05:19:28.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:28.545593 8328 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:19:28.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 05:19:28 dist-test-slave-lhv2 krb5kdc[8315](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790572768, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.27.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:19:28.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:28.552265 8328 init.cc:377] Logged in from keytab as kudu/127.8.27.254@KRBTEST.COM (short username kudu)
05:19:28.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:28.557893 8328 webserver.cc:460] Webserver started at http://127.8.27.254:35839/ using document root <none> and password file <none>
05:19:28.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:28.558843 8328 fs_manager.cc:359] Metadata directory not provided
05:19:28.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:28.559077 8328 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:19:28.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:28.559669 8328 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:19:28.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:28.564630 8328 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-0/data/instance:
05:19:28.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "068ae5f7ad434c4291c050390e0b2580"
05:19:28.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 05:19:28 on dist-test-slave-lhv2"
05:19:28.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:28.565790 8328 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-0/wal/instance:
05:19:28.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "068ae5f7ad434c4291c050390e0b2580"
05:19:28.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 05:19:28 on dist-test-slave-lhv2"
05:19:28.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:28.573762 8328 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.004s
05:19:28.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:28.579631 8346 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:19:28.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:28.580950 8328 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
05:19:28.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:28.581310 8328 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-0/data,/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-0/wal
05:19:28.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "068ae5f7ad434c4291c050390e0b2580"
05:19:28.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 05:19:28 on dist-test-slave-lhv2"
05:19:28.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:28.581686 8328 fs_report.cc:383] FS layout report
05:19:28.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:19:28.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-0/wal
05:19:28.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-0/wal
05:19:28.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-0/data/data
05:19:28.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:19:28.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:19:28.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:19:28.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:19:28.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:19:28.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:19:28.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:19:28.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:19:28.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:19:28.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:19:28.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:19:28.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:28.629341 8328 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:19:28.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:28.636845 8328 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:19:28.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:28.637475 8328 kserver.cc:163] Server-wide thread pool size limit: 3276
05:19:28.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:28.720443 8328 rpc_server.cc:307] RPC server started. Bound to: 127.8.27.254:34253
05:19:28.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:28.720470 8397 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.8.27.254:34253 every 8 connection(s)
05:19:28.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:28.723829 8328 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-0/data/info.pb
05:19:28.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:28.730505 8398 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
05:19:28.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:28.734153 8303 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskHSJIN2/build/tsan/bin/kudu as pid 8328
05:19:28.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:28.734702 8303 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-0/wal/instance
05:19:28.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:28.743558 8398 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.27.254" port: 34253 } has no permanent_uuid. Determining permanent_uuid...
05:19:28.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.8.27.253@KRBTEST.COM; defaulting to no policy
05:19:28.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
05:19:28.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.8.27.253@KRBTEST.COM" created.
05:19:28.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 05:19:28 dist-test-slave-lhv2 krb5kdc[8315](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790572768, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.27.254@KRBTEST.COM for kudu/127.8.27.254@KRBTEST.COM
05:19:28.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
05:19:28.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.8.27.253 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/krb5kdc/kudu_127.8.27.253.keytab.
05:19:28.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.8.27.253 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/krb5kdc/kudu_127.8.27.253.keytab.
05:19:28.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:28.788005 8303 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskHSJIN2/build/tsan/bin/kudu
05:19:28.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskHSJIN2/build/tsan/bin/kudu
05:19:28.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-1/wal
05:19:28.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-1/data
05:19:28.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:19:28.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:19:28.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:19:28.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:19:28.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:19:28.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:19:28.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-1/logs
05:19:28.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-1/data/info.pb
05:19:28.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:19:28.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:19:28.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:19:28.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:19:28.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:19:28.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:19:28.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:19:28.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:19:28.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:19:28.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:19:28.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:19:28.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:19:28.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.27.253:44545
05:19:28.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.27.253
05:19:28.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:19:28.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/krb5kdc/kudu_127.8.27.253.keytab
05:19:28.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.27.253
05:19:28.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:19:28.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
05:19:28.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
05:19:28.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.27.212:40337
05:19:28.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:19:28.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:19:28.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:19:28.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:19:28.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.27.254:34253,127.8.27.253:44545,127.8.27.252:39249
05:19:28.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:19:28.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37289/jwks.json
05:19:28.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
05:19:28.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:28.792290 8398 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.27.253" port: 44545 } has no permanent_uuid. Determining permanent_uuid...
05:19:28.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:28.795034 8349 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.27.253:44545: connect: Connection refused (error 111)
05:19:28.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:28.799611 8398 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.27.253:44545: Network error: Client connection negotiation failed: client connection to 127.8.27.253:44545: connect: Connection refused (error 111)
05:19:28.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:28.849141 8398 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.27.253" port: 44545 } attempt: 1
05:19:28.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:28.853248 8398 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.27.253:44545: Network error: Client connection negotiation failed: client connection to 127.8.27.253:44545: connect: Connection refused (error 111)
05:19:28.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:28.921705 8398 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.27.253" port: 44545 } attempt: 2
05:19:28.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:28.925843 8398 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.27.253:44545: Network error: Client connection negotiation failed: client connection to 127.8.27.253:44545: connect: Connection refused (error 111)
05:19:29.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:29.017256 8398 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.27.253" port: 44545 } attempt: 3
05:19:29.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:29.021838 8398 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.27.253:44545: Network error: Client connection negotiation failed: client connection to 127.8.27.253:44545: connect: Connection refused (error 111)
05:19:29.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:29.111784 8404 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:19:29.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:29.112473 8404 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:19:29.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:29.113170 8404 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:19:29.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:29.145077 8404 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
05:19:29.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:29.145370 8404 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:19:29.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:29.145750 8404 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:19:29.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:29.146068 8404 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:19:29.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:29.146391 8404 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:19:29.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:29.146704 8404 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:19:29.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:29.146953 8404 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37289/jwks.json
05:19:29.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:29.165211 8398 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.27.253" port: 44545 } attempt: 4
05:19:29.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:29.169911 8398 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.27.253:44545: Network error: Client connection negotiation failed: client connection to 127.8.27.253:44545: connect: Connection refused (error 111)
05:19:29.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:29.185860 8404 master_runner.cc:386] Master server non-default flags:
05:19:29.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:19:29.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.27.212:40337
05:19:29.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:19:29.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:19:29.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-1/data
05:19:29.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-1/wal
05:19:29.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
05:19:29.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:19:29.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.27.254:34253,127.8.27.253:44545,127.8.27.252:39249
05:19:29.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/krb5kdc/kudu_127.8.27.253.keytab
05:19:29.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.27.253
05:19:29.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:19:29.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:19:29.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:19:29.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.27.253:44545
05:19:29.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:19:29.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:19:29.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:19:29.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37289/jwks.json
05:19:29.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:19:29.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
05:19:29.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
05:19:29.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:19:29.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:19:29.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-1/data/info.pb
05:19:29.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.27.253
05:19:29.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:19:29.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:19:29.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:19:29.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:19:29.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:19:29.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:19:29.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-1/logs
05:19:29.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:19:29.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:19:29.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:19:29.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:19:29.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:19:29.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:19:29.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:19:29.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 04:35:01 UTC on ccycloud.quasar-alllgi.root.comops.site
05:19:29.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676563
05:19:29.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:19:29.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:29.187373 8404 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:19:29.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:29.188818 8404 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:19:29.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:29.208853 8412 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:19:29.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:29.469352 8398 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.27.253" port: 44545 } attempt: 5
05:19:29.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:29.473212 8398 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.27.253:44545: Network error: Client connection negotiation failed: client connection to 127.8.27.253:44545: connect: Connection refused (error 111)
05:19:29.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:29.529176 8288 debug-util.cc:398] Leaking SignalData structure 0x7b08000b4060 after lost signal to thread 8163
05:19:29.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:29.530913 8288 debug-util.cc:398] Leaking SignalData structure 0x7b08000bdec0 after lost signal to thread 8291
05:19:30.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:30.020761 8398 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.27.253" port: 44545 } attempt: 6
05:19:30.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:30.050601 8398 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.27.253:44545: Network error: Client connection negotiation failed: client connection to 127.8.27.253:44545: connect: Connection refused (error 111)
05:19:30.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:29.208997 8411 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:19:30.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:29.209659 8414 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:19:30.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:29.212081 8404 server_base.cc:1034] running on GCE node
05:19:30.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:30.537917 8404 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:19:30.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:30.541677 8404 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:19:30.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:30.543358 8404 hybrid_clock.cc:648] HybridClock initialized: now 1790572770543282 us; error 90 us; skew 500 ppm
05:19:30.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:30.544432 8404 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:19:30.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 05:19:30 dist-test-slave-lhv2 krb5kdc[8315](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790572770, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.27.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:19:30.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:30.557927 8404 init.cc:377] Logged in from keytab as kudu/127.8.27.253@KRBTEST.COM (short username kudu)
05:19:30.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:30.564352 8404 webserver.cc:460] Webserver started at http://127.8.27.253:45179/ using document root <none> and password file <none>
05:19:30.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:30.565411 8404 fs_manager.cc:359] Metadata directory not provided
05:19:30.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:30.565650 8404 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:19:30.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:30.566188 8404 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:19:30.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:30.571707 8404 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-1/data/instance:
05:19:30.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a5a0acd5ec4048e38717267b9d89fdfb"
05:19:30.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 05:19:30 on dist-test-slave-lhv2"
05:19:30.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:30.573261 8404 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-1/wal/instance:
05:19:30.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a5a0acd5ec4048e38717267b9d89fdfb"
05:19:30.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 05:19:30 on dist-test-slave-lhv2"
05:19:30.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:30.581163 8404 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.004s
05:19:30.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:30.587566 8426 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:19:30.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:30.588901 8404 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
05:19:30.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:30.589310 8404 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-1/data,/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-1/wal
05:19:30.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a5a0acd5ec4048e38717267b9d89fdfb"
05:19:30.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 05:19:30 on dist-test-slave-lhv2"
05:19:30.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:30.589769 8404 fs_report.cc:383] FS layout report
05:19:30.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:19:30.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-1/wal
05:19:30.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-1/wal
05:19:30.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-1/data/data
05:19:30.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:19:30.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:19:30.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:19:30.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:19:30.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:19:30.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:19:30.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:19:30.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:19:30.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:19:30.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:19:30.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:19:30.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:30.635000 8404 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:19:30.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:30.643610 8404 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:19:30.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:30.644151 8404 kserver.cc:163] Server-wide thread pool size limit: 3276
05:19:30.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:30.751024 8404 rpc_server.cc:307] RPC server started. Bound to: 127.8.27.253:44545
05:19:30.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:30.751200 8477 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.8.27.253:44545 every 8 connection(s)
05:19:30.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:30.753937 8404 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-1/data/info.pb
05:19:30.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:30.761134 8478 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
05:19:30.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:30.761328 8303 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskHSJIN2/build/tsan/bin/kudu as pid 8404
05:19:30.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:30.761782 8303 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-1/wal/instance
05:19:30.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.8.27.252@KRBTEST.COM; defaulting to no policy
05:19:30.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:30.780515 8478 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.27.254" port: 34253 } has no permanent_uuid. Determining permanent_uuid...
05:19:30.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
05:19:30.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.8.27.252@KRBTEST.COM" created.
05:19:30.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
05:19:30.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.8.27.252 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/krb5kdc/kudu_127.8.27.252.keytab.
05:19:30.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.8.27.252 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/krb5kdc/kudu_127.8.27.252.keytab.
05:19:30.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 05:19:30 dist-test-slave-lhv2 krb5kdc[8315](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790572770, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.27.253@KRBTEST.COM for kudu/127.8.27.254@KRBTEST.COM
05:19:30.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:30.832934 8303 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskHSJIN2/build/tsan/bin/kudu
05:19:30.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskHSJIN2/build/tsan/bin/kudu
05:19:30.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-2/wal
05:19:30.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-2/data
05:19:30.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:19:30.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:19:30.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:19:30.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:19:30.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:19:30.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:19:30.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-2/logs
05:19:30.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-2/data/info.pb
05:19:30.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:19:30.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:19:30.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:19:30.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:19:30.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:19:30.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:19:30.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:19:30.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:19:30.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:19:30.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:19:30.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:19:30.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:19:30.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.27.252:39249
05:19:30.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.27.252
05:19:30.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:19:30.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/krb5kdc/kudu_127.8.27.252.keytab
05:19:30.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.27.252
05:19:30.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:19:30.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
05:19:30.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
05:19:30.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.27.212:40337
05:19:30.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:19:30.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:19:30.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:19:30.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:19:30.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.27.254:34253,127.8.27.253:44545,127.8.27.252:39249
05:19:30.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:19:30.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37289/jwks.json
05:19:30.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
05:19:30.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:30.849574 8478 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.27.253" port: 44545 } has no permanent_uuid. Determining permanent_uuid...
05:19:30.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 05:19:30 dist-test-slave-lhv2 krb5kdc[8315](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790572770, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.27.253@KRBTEST.COM for kudu/127.8.27.253@KRBTEST.COM
05:19:30.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:30.888140 8478 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.27.252" port: 39249 } has no permanent_uuid. Determining permanent_uuid...
05:19:30.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:30.890599 8429 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.27.252:39249: connect: Connection refused (error 111)
05:19:30.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:30.895324 8478 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.27.252:39249: Network error: Client connection negotiation failed: client connection to 127.8.27.252:39249: connect: Connection refused (error 111)
05:19:30.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:30.945961 8478 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.27.252" port: 39249 } attempt: 1
05:19:30.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:30.953698 8478 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.27.252:39249: Network error: Client connection negotiation failed: client connection to 127.8.27.252:39249: connect: Connection refused (error 111)
05:19:31.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:31.022142 8478 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.27.252" port: 39249 } attempt: 2
05:19:31.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:31.025655 8478 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.27.252:39249: Network error: Client connection negotiation failed: client connection to 127.8.27.252:39249: connect: Connection refused (error 111)
05:19:31.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:31.111042 8398 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.27.253" port: 44545 } attempt: 7
05:19:31.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:31.117048 8478 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.27.252" port: 39249 } attempt: 3
05:19:31.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:31.121855 8478 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.27.252:39249: Network error: Client connection negotiation failed: client connection to 127.8.27.252:39249: connect: Connection refused (error 111)
05:19:31.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 05:19:31 dist-test-slave-lhv2 krb5kdc[8315](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790572768, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.27.254@KRBTEST.COM for kudu/127.8.27.253@KRBTEST.COM
05:19:31.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:31.136642 8398 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.27.252" port: 39249 } has no permanent_uuid. Determining permanent_uuid...
05:19:31.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:31.142032 8398 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.27.252:39249: Network error: Client connection negotiation failed: client connection to 127.8.27.252:39249: connect: Connection refused (error 111)
05:19:31.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:31.200491 8398 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.27.252" port: 39249 } attempt: 1
05:19:31.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:31.204600 8398 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.27.252:39249: Network error: Client connection negotiation failed: client connection to 127.8.27.252:39249: connect: Connection refused (error 111)
05:19:31.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:31.263695 8484 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:19:31.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:31.264400 8484 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:19:31.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:31.265221 8484 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:19:31.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:31.265291 8478 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.27.252" port: 39249 } attempt: 4
05:19:31.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:31.270166 8478 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.27.252:39249: Network error: Client connection negotiation failed: client connection to 127.8.27.252:39249: connect: Connection refused (error 111)
05:19:31.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:31.285995 8398 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.27.252" port: 39249 } attempt: 2
05:19:31.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:31.290803 8398 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.27.252:39249: Network error: Client connection negotiation failed: client connection to 127.8.27.252:39249: connect: Connection refused (error 111)
05:19:31.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:31.300647 8484 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
05:19:31.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:31.300926 8484 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:19:31.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:31.301309 8484 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:19:31.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:31.301636 8484 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:19:31.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:31.301945 8484 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:19:31.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:31.302341 8484 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:19:31.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:31.302574 8484 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37289/jwks.json
05:19:31.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:31.342473 8484 master_runner.cc:386] Master server non-default flags:
05:19:31.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:19:31.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.27.212:40337
05:19:31.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:19:31.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:19:31.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-2/data
05:19:31.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-2/wal
05:19:31.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
05:19:31.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:19:31.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.27.254:34253,127.8.27.253:44545,127.8.27.252:39249
05:19:31.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/krb5kdc/kudu_127.8.27.252.keytab
05:19:31.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.27.252
05:19:31.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:19:31.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:19:31.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:19:31.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.27.252:39249
05:19:31.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:19:31.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:19:31.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:19:31.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37289/jwks.json
05:19:31.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:19:31.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
05:19:31.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
05:19:31.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:19:31.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:19:31.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-2/data/info.pb
05:19:31.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.27.252
05:19:31.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:19:31.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:19:31.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:19:31.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:19:31.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:19:31.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:19:31.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-2/logs
05:19:31.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:19:31.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:19:31.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:19:31.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:19:31.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:19:31.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:19:31.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:19:31.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 04:35:01 UTC on ccycloud.quasar-alllgi.root.comops.site
05:19:31.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676563
05:19:31.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:19:31.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:31.344079 8484 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:19:31.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:31.345290 8484 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:19:31.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:31.369195 8496 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:19:31.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:31.376338 8398 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.27.252" port: 39249 } attempt: 3
05:19:31.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:31.380998 8398 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.27.252:39249: Network error: Client connection negotiation failed: client connection to 127.8.27.252:39249: connect: Connection refused (error 111)
05:19:31.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:31.521502 8398 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.27.252" port: 39249 } attempt: 4
05:19:31.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:31.527328 8398 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.27.252:39249: Network error: Client connection negotiation failed: client connection to 127.8.27.252:39249: connect: Connection refused (error 111)
05:19:31.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:31.569635 8478 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.27.252" port: 39249 } attempt: 5
05:19:31.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:31.575222 8478 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.27.252:39249: Network error: Client connection negotiation failed: client connection to 127.8.27.252:39249: connect: Connection refused (error 111)
05:19:31.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:31.810801 8398 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.27.252" port: 39249 } attempt: 5
05:19:31.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:31.816525 8398 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.27.252:39249: Network error: Client connection negotiation failed: client connection to 127.8.27.252:39249: connect: Connection refused (error 111)
05:19:32.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:32.122674 8478 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.27.252" port: 39249 } attempt: 6
05:19:32.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:32.130879 8478 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.27.252:39249: Network error: Client connection negotiation failed: client connection to 127.8.27.252:39249: connect: Connection refused (error 111)
05:19:32.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:32.368943 8398 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.27.252" port: 39249 } attempt: 6
05:19:32.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:32.376268 8398 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.27.252:39249: Network error: Client connection negotiation failed: client connection to 127.8.27.252:39249: connect: Connection refused (error 111)
05:19:32.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:31.372650 8498 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:19:32.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:31.370378 8495 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:19:32.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:32.655206 8484 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
05:19:32.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:32.654760 8497 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
05:19:32.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:32.658665 8484 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:19:32.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:32.662000 8484 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:19:32.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:32.664359 8484 hybrid_clock.cc:648] HybridClock initialized: now 1790572772664314 us; error 51 us; skew 500 ppm
05:19:32.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:32.665252 8484 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:19:32.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 05:19:32 dist-test-slave-lhv2 krb5kdc[8315](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790572772, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.27.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:19:32.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:32.672668 8484 init.cc:377] Logged in from keytab as kudu/127.8.27.252@KRBTEST.COM (short username kudu)
05:19:32.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:32.679016 8484 webserver.cc:460] Webserver started at http://127.8.27.252:41723/ using document root <none> and password file <none>
05:19:32.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:32.680012 8484 fs_manager.cc:359] Metadata directory not provided
05:19:32.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:32.680207 8484 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:19:32.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:32.680625 8484 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:19:32.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:32.685434 8484 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-2/data/instance:
05:19:32.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5f42df00f0b34da18c6f5659ce854d38"
05:19:32.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 05:19:32 on dist-test-slave-lhv2"
05:19:32.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:32.686514 8484 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-2/wal/instance:
05:19:32.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5f42df00f0b34da18c6f5659ce854d38"
05:19:32.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 05:19:32 on dist-test-slave-lhv2"
05:19:32.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:32.694037 8484 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.004s
05:19:32.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:32.699657 8513 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:19:32.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:32.700941 8484 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
05:19:32.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:32.701314 8484 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-2/data,/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-2/wal
05:19:32.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5f42df00f0b34da18c6f5659ce854d38"
05:19:32.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 05:19:32 on dist-test-slave-lhv2"
05:19:32.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:32.701630 8484 fs_report.cc:383] FS layout report
05:19:32.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:19:32.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-2/wal
05:19:32.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-2/wal
05:19:32.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-2/data/data
05:19:32.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:19:32.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:19:32.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:19:32.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:19:32.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:19:32.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:19:32.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:19:32.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:19:32.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:19:32.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:19:32.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:19:32.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:32.742558 8484 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:19:32.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:32.751108 8484 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:19:32.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:32.751740 8484 kserver.cc:163] Server-wide thread pool size limit: 3276
05:19:32.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:32.854249 8484 rpc_server.cc:307] RPC server started. Bound to: 127.8.27.252:39249
05:19:32.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:32.854389 8564 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.8.27.252:39249 every 8 connection(s)
05:19:32.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:32.857060 8484 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-2/data/info.pb
05:19:32.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:32.857913 8303 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskHSJIN2/build/tsan/bin/kudu as pid 8484
05:19:32.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:32.858374 8303 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-2/wal/instance
05:19:32.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:32.864493 8565 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
05:19:32.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:32.879601 8565 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.27.254" port: 34253 } has no permanent_uuid. Determining permanent_uuid...
05:19:32.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 05:19:32 dist-test-slave-lhv2 krb5kdc[8315](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790572766, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.8.27.252@KRBTEST.COM
05:19:32.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 05:19:32 dist-test-slave-lhv2 krb5kdc[8315](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790572772, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.27.252@KRBTEST.COM for kudu/127.8.27.254@KRBTEST.COM
05:19:32.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:32.917439 8565 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.27.253" port: 44545 } has no permanent_uuid. Determining permanent_uuid...
05:19:32.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 05:19:32 dist-test-slave-lhv2 krb5kdc[8315](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790572772, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.27.252@KRBTEST.COM for kudu/127.8.27.253@KRBTEST.COM
05:19:32.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 05:19:32 dist-test-slave-lhv2 krb5kdc[8315](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790572766, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.8.27.253@KRBTEST.COM
05:19:32.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:32.946954 8565 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.27.252" port: 39249 } has no permanent_uuid. Determining permanent_uuid...
05:19:32.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 05:19:32 dist-test-slave-lhv2 krb5kdc[8315](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790572772, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.27.252@KRBTEST.COM for kudu/127.8.27.252@KRBTEST.COM
05:19:32.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 05:19:32 dist-test-slave-lhv2 krb5kdc[8315](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790572766, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.8.27.254@KRBTEST.COM
05:19:32.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:32.983213 8303 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
05:19:32.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:32.984638 8565 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38: Bootstrap starting.
05:19:32.991 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:19:32.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:32.991883 8565 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38: Neither blocks nor log segments found. Creating new log.
05:19:32.992 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
05:19:32.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:32.994051 8565 log.cc:824] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38: Log is configured to *not* fsync() on all Append() calls
05:19:33.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:32.999845 8565 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38: No bootstrap required, opened a new log
05:19:33.004 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
05:19:33.005 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
05:19:33.009 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.8.27.254"
port: 34253
}
bound_http_address {
host: "127.8.27.254"
port: 35839
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.8.27.253"
port: 44545
}
bound_http_address {
host: "127.8.27.253"
port: 45179
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.8.27.252"
port: 39249
}
bound_http_address {
host: "127.8.27.252"
port: 41723
}
}
}
05:19:33.010 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
05:19:33.011 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:19:33.011 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
05:19:33.013 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
05:19:33.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:33.028209 8565 raft_consensus.cc:348] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "068ae5f7ad434c4291c050390e0b2580" member_type: VOTER last_known_addr { host: "127.8.27.254" port: 34253 } } peers { permanent_uuid: "a5a0acd5ec4048e38717267b9d89fdfb" member_type: VOTER last_known_addr { host: "127.8.27.253" port: 44545 } } peers { permanent_uuid: "5f42df00f0b34da18c6f5659ce854d38" member_type: VOTER last_known_addr { host: "127.8.27.252" port: 39249 } }
05:19:33.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:33.029229 8565 raft_consensus.cc:374] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
05:19:33.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:33.029563 8565 raft_consensus.cc:729] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5f42df00f0b34da18c6f5659ce854d38, State: Initialized, Role: FOLLOWER
05:19:33.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:33.030582 8565 consensus_queue.cc:260] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "068ae5f7ad434c4291c050390e0b2580" member_type: VOTER last_known_addr { host: "127.8.27.254" port: 34253 } } peers { permanent_uuid: "a5a0acd5ec4048e38717267b9d89fdfb" member_type: VOTER last_known_addr { host: "127.8.27.253" port: 44545 } } peers { permanent_uuid: "5f42df00f0b34da18c6f5659ce854d38" member_type: VOTER last_known_addr { host: "127.8.27.252" port: 39249 } }
05:19:33.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:33.034284 8575 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "068ae5f7ad434c4291c050390e0b2580" member_type: VOTER last_known_addr { host: "127.8.27.254" port: 34253 } } peers { permanent_uuid: "a5a0acd5ec4048e38717267b9d89fdfb" member_type: VOTER last_known_addr { host: "127.8.27.253" port: 44545 } } peers { permanent_uuid: "5f42df00f0b34da18c6f5659ce854d38" member_type: VOTER last_known_addr { host: "127.8.27.252" port: 39249 } } }
05:19:33.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:33.035028 8575 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38 [sys.catalog]: This master's current role is: FOLLOWER
05:19:33.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:33.035354 8565 sys_catalog.cc:564] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38 [sys.catalog]: configured and running, proceeding with master startup.
05:19:33.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 05:19:33 dist-test-slave-lhv2 krb5kdc[8315](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790572773, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:19:33.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
05:19:33.054 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:19:33.057 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/krb5kdc/krb5cc
05:19:33.066 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
05:19:33.068 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/krb5kdc/krb5cc
05:19:33.069 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
05:19:33.073 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
05:19:33.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:33.074951 8588 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:19:33.075 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
05:19:33.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:33.075371 8588 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:19:33.076 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
05:19:33.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:33.069496 8484 master_runner.cc:186] Error getting master registration for 127.8.27.254:34253: OK, instance_id { permanent_uuid: "068ae5f7ad434c4291c050390e0b2580" instance_seqno: 1790572768697645 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
05:19:33.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:33.097234 8484 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:19:33.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 05:19:33 dist-test-slave-lhv2 krb5kdc[8315](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790572773, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:19:33.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
05:19:33.115 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:19:33.115 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/krb5kdc/krb5cc
05:19:33.117 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
05:19:33.118 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.8.27.252:39249
05:19:33.120 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
05:19:33.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:33.121174 8303 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskHSJIN2/build/tsan/bin/kudu with pid 8484
05:19:33.138 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:19:33.139 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.8.27.253:44545
05:19:33.140 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
05:19:33.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:33.140488 8303 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskHSJIN2/build/tsan/bin/kudu with pid 8404
05:19:33.164 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:19:33.164 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.8.27.254:34253
05:19:33.165 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
05:19:33.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:33.165387 8303 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskHSJIN2/build/tsan/bin/kudu with pid 8328
05:19:33.188 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:19:33.189 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.8.27.252:39249
05:19:33.190 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
05:19:33.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:33.192065 8303 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskHSJIN2/build/tsan/bin/kudu
05:19:33.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskHSJIN2/build/tsan/bin/kudu
05:19:33.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-2/wal
05:19:33.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-2/data
05:19:33.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:19:33.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:19:33.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:19:33.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:19:33.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:19:33.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:19:33.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-2/logs
05:19:33.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-2/data/info.pb
05:19:33.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:19:33.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:19:33.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:19:33.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:19:33.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:19:33.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:19:33.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:19:33.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:19:33.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:19:33.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:19:33.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:19:33.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:19:33.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.27.252:39249
05:19:33.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.27.252
05:19:33.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41723
05:19:33.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/krb5kdc/kudu_127.8.27.252.keytab
05:19:33.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.27.252
05:19:33.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:19:33.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
05:19:33.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
05:19:33.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.27.212:40337
05:19:33.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:19:33.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:19:33.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:19:33.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:19:33.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.27.254:34253,127.8.27.253:44545,127.8.27.252:39249
05:19:33.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:19:33.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37289/jwks.json
05:19:33.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
05:19:33.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:33.501942 8592 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:19:33.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:33.502478 8592 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:19:33.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:33.503043 8592 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:19:33.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:33.542004 8592 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
05:19:33.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:33.542320 8592 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:19:33.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:33.542768 8592 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:19:33.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:33.543088 8592 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:19:33.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:33.543387 8592 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:19:33.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:33.543691 8592 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:19:33.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:33.543936 8592 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37289/jwks.json
05:19:33.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:33.584381 8592 master_runner.cc:386] Master server non-default flags:
05:19:33.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:19:33.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.27.212:40337
05:19:33.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:19:33.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:19:33.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-2/data
05:19:33.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-2/wal
05:19:33.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
05:19:33.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:19:33.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.27.254:34253,127.8.27.253:44545,127.8.27.252:39249
05:19:33.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/krb5kdc/kudu_127.8.27.252.keytab
05:19:33.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.27.252
05:19:33.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:19:33.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:19:33.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:19:33.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.27.252:39249
05:19:33.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:19:33.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:19:33.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:19:33.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37289/jwks.json
05:19:33.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:19:33.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
05:19:33.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
05:19:33.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:19:33.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:19:33.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-2/data/info.pb
05:19:33.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.27.252
05:19:33.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41723
05:19:33.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:19:33.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:19:33.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:19:33.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:19:33.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:19:33.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-2/logs
05:19:33.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:19:33.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:19:33.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:19:33.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:19:33.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:19:33.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:19:33.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:19:33.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 04:35:01 UTC on ccycloud.quasar-alllgi.root.comops.site
05:19:33.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676563
05:19:33.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:19:33.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:33.585904 8592 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:19:33.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:33.587373 8592 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:19:33.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:33.604825 8598 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:19:34.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:33.604821 8599 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:19:34.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:33.605691 8601 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:19:34.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:33.608327 8592 server_base.cc:1034] running on GCE node
05:19:34.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:34.867230 8592 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:19:34.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:34.869962 8592 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:19:34.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:34.871364 8592 hybrid_clock.cc:648] HybridClock initialized: now 1790572774871298 us; error 58 us; skew 500 ppm
05:19:34.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:34.872221 8592 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:19:34.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 05:19:34 dist-test-slave-lhv2 krb5kdc[8315](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790572774, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.27.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:19:34.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:34.878594 8592 init.cc:377] Logged in from keytab as kudu/127.8.27.252@KRBTEST.COM (short username kudu)
05:19:34.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:34.883363 8592 webserver.cc:460] Webserver started at http://127.8.27.252:41723/ using document root <none> and password file <none>
05:19:34.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:34.884228 8592 fs_manager.cc:359] Metadata directory not provided
05:19:34.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:34.884425 8592 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:19:34.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:34.892292 8592 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.003s sys 0.001s
05:19:34.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:34.896684 8610 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:19:34.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:34.897882 8592 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.001s
05:19:34.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:34.898219 8592 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-2/data,/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-2/wal
05:19:34.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5f42df00f0b34da18c6f5659ce854d38"
05:19:34.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 05:19:32 on dist-test-slave-lhv2"
05:19:34.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:34.900583 8592 fs_report.cc:383] FS layout report
05:19:34.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:19:34.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-2/wal
05:19:34.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-2/wal
05:19:34.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-2/data/data
05:19:34.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:19:34.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:19:34.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:19:34.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:19:34.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:19:34.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:19:34.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:19:34.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:19:34.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:19:34.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:19:34.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:19:34.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:34.943833 8592 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:19:34.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:34.949695 8592 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:19:34.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:34.950181 8592 kserver.cc:163] Server-wide thread pool size limit: 3276
05:19:35.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:35.030737 8592 rpc_server.cc:307] RPC server started. Bound to: 127.8.27.252:39249
05:19:35.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:35.030874 8661 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.8.27.252:39249 every 8 connection(s)
05:19:35.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:35.033520 8592 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-2/data/info.pb
05:19:35.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:35.039700 8303 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskHSJIN2/build/tsan/bin/kudu as pid 8592
05:19:35.040 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:19:35.040 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.8.27.253:44545
05:19:35.040 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
05:19:35.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:35.042035 8303 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskHSJIN2/build/tsan/bin/kudu
05:19:35.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskHSJIN2/build/tsan/bin/kudu
05:19:35.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-1/wal
05:19:35.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-1/data
05:19:35.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:19:35.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:19:35.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:19:35.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:19:35.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:19:35.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:19:35.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-1/logs
05:19:35.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-1/data/info.pb
05:19:35.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:19:35.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:19:35.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:19:35.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:19:35.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:19:35.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:19:35.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:19:35.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:19:35.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:19:35.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:19:35.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:19:35.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:19:35.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.27.253:44545
05:19:35.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.27.253
05:19:35.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45179
05:19:35.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/krb5kdc/kudu_127.8.27.253.keytab
05:19:35.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.27.253
05:19:35.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:19:35.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
05:19:35.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
05:19:35.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.27.212:40337
05:19:35.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:19:35.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:19:35.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:19:35.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:19:35.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.27.254:34253,127.8.27.253:44545,127.8.27.252:39249
05:19:35.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:19:35.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37289/jwks.json
05:19:35.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
05:19:35.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:35.044435 8662 sys_catalog.cc:263] Verifying existing consensus state
05:19:35.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:35.051625 8662 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38: Bootstrap starting.
05:19:35.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:35.097481 8662 log.cc:824] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38: Log is configured to *not* fsync() on all Append() calls
05:19:35.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:35.102439 8662 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
05:19:35.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:35.103322 8662 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38: Bootstrap complete.
05:19:35.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:35.138958 8662 raft_consensus.cc:348] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "068ae5f7ad434c4291c050390e0b2580" member_type: VOTER last_known_addr { host: "127.8.27.254" port: 34253 } } peers { permanent_uuid: "a5a0acd5ec4048e38717267b9d89fdfb" member_type: VOTER last_known_addr { host: "127.8.27.253" port: 44545 } } peers { permanent_uuid: "5f42df00f0b34da18c6f5659ce854d38" member_type: VOTER last_known_addr { host: "127.8.27.252" port: 39249 } }
05:19:35.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:35.142459 8662 raft_consensus.cc:374] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
05:19:35.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:35.142795 8662 raft_consensus.cc:729] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5f42df00f0b34da18c6f5659ce854d38, State: Initialized, Role: FOLLOWER
05:19:35.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:35.143782 8662 consensus_queue.cc:260] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "068ae5f7ad434c4291c050390e0b2580" member_type: VOTER last_known_addr { host: "127.8.27.254" port: 34253 } } peers { permanent_uuid: "a5a0acd5ec4048e38717267b9d89fdfb" member_type: VOTER last_known_addr { host: "127.8.27.253" port: 44545 } } peers { permanent_uuid: "5f42df00f0b34da18c6f5659ce854d38" member_type: VOTER last_known_addr { host: "127.8.27.252" port: 39249 } }
05:19:35.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:35.146638 8666 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "068ae5f7ad434c4291c050390e0b2580" member_type: VOTER last_known_addr { host: "127.8.27.254" port: 34253 } } peers { permanent_uuid: "a5a0acd5ec4048e38717267b9d89fdfb" member_type: VOTER last_known_addr { host: "127.8.27.253" port: 44545 } } peers { permanent_uuid: "5f42df00f0b34da18c6f5659ce854d38" member_type: VOTER last_known_addr { host: "127.8.27.252" port: 39249 } } }
05:19:35.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:35.147434 8666 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38 [sys.catalog]: This master's current role is: FOLLOWER
05:19:35.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:35.148496 8662 sys_catalog.cc:564] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38 [sys.catalog]: configured and running, proceeding with master startup.
05:19:35.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:35.178145 8677 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:19:35.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:35.178869 8677 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:19:35.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:35.206758 8616 tablet.cc:2369] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38: Can't schedule compaction. Clean time has not been advanced past its initial value.
05:19:35.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:35.185084 8592 master_runner.cc:186] Error getting master registration for 127.8.27.254:34253: Network error: Client connection negotiation failed: client connection to 127.8.27.254:34253: connect: Connection refused (error 111),
05:19:35.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:35.216298 8592 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:19:35.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:35.419760 8664 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:19:35.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:35.421177 8664 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:19:35.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:35.422021 8664 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:19:35.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:35.456070 8664 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
05:19:35.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:35.456357 8664 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:19:35.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:35.456780 8664 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:19:35.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:35.457091 8664 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:19:35.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:35.457436 8664 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:19:35.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:35.457753 8664 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:19:35.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:35.457994 8664 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37289/jwks.json
05:19:35.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:35.498113 8664 master_runner.cc:386] Master server non-default flags:
05:19:35.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:19:35.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.27.212:40337
05:19:35.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:19:35.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:19:35.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-1/data
05:19:35.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-1/wal
05:19:35.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
05:19:35.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:19:35.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.27.254:34253,127.8.27.253:44545,127.8.27.252:39249
05:19:35.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/krb5kdc/kudu_127.8.27.253.keytab
05:19:35.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.27.253
05:19:35.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:19:35.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:19:35.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:19:35.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.27.253:44545
05:19:35.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:19:35.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:19:35.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:19:35.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37289/jwks.json
05:19:35.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:19:35.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
05:19:35.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
05:19:35.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:19:35.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:19:35.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-1/data/info.pb
05:19:35.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.27.253
05:19:35.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45179
05:19:35.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:19:35.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:19:35.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:19:35.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:19:35.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:19:35.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-1/logs
05:19:35.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:19:35.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:19:35.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:19:35.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:19:35.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:19:35.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:19:35.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:19:35.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 04:35:01 UTC on ccycloud.quasar-alllgi.root.comops.site
05:19:35.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676563
05:19:35.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:19:35.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:35.499712 8664 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:19:35.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:35.500962 8664 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:19:35.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:35.520902 8687 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:19:35.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:35.530871 8666 raft_consensus.cc:482] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
05:19:35.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:35.531523 8666 raft_consensus.cc:504] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "068ae5f7ad434c4291c050390e0b2580" member_type: VOTER last_known_addr { host: "127.8.27.254" port: 34253 } } peers { permanent_uuid: "a5a0acd5ec4048e38717267b9d89fdfb" member_type: VOTER last_known_addr { host: "127.8.27.253" port: 44545 } } peers { permanent_uuid: "5f42df00f0b34da18c6f5659ce854d38" member_type: VOTER last_known_addr { host: "127.8.27.252" port: 39249 } }
05:19:35.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:35.535779 8614 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.27.254:34253: connect: Connection refused (error 111)
05:19:35.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:35.537956 8666 leader_election.cc:288] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 068ae5f7ad434c4291c050390e0b2580 (127.8.27.254:34253), a5a0acd5ec4048e38717267b9d89fdfb (127.8.27.253:44545)
05:19:35.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:35.541132 8614 leader_election.cc:334] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 068ae5f7ad434c4291c050390e0b2580 (127.8.27.254:34253): Network error: Client connection negotiation failed: client connection to 127.8.27.254:34253: connect: Connection refused (error 111)
05:19:35.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:35.543883 8613 leader_election.cc:334] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a5a0acd5ec4048e38717267b9d89fdfb (127.8.27.253:44545): Network error: Client connection negotiation failed: client connection to 127.8.27.253:44545: connect: Connection refused (error 111)
05:19:35.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:35.544281 8613 leader_election.cc:302] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5f42df00f0b34da18c6f5659ce854d38; no voters: 068ae5f7ad434c4291c050390e0b2580, a5a0acd5ec4048e38717267b9d89fdfb
05:19:35.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:35.545109 8666 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
05:19:36.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:36.183355 8677 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:19:36.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:36.183700 8677 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:19:36.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:35.523303 8689 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:19:36.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:35.522527 8664 server_base.cc:1034] running on GCE node
05:19:36.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:35.521775 8686 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:19:36.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:36.831872 8664 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:19:36.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:36.837549 8664 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:19:36.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:36.839275 8664 hybrid_clock.cc:648] HybridClock initialized: now 1790572776839221 us; error 43 us; skew 500 ppm
05:19:36.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:36.840233 8664 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:19:36.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 05:19:36 dist-test-slave-lhv2 krb5kdc[8315](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790572776, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.27.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:19:36.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:36.851517 8664 init.cc:377] Logged in from keytab as kudu/127.8.27.253@KRBTEST.COM (short username kudu)
05:19:36.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:36.857538 8664 webserver.cc:460] Webserver started at http://127.8.27.253:45179/ using document root <none> and password file <none>
05:19:36.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:36.858472 8664 fs_manager.cc:359] Metadata directory not provided
05:19:36.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:36.858675 8664 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:19:36.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:36.867676 8664 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.007s sys 0.000s
05:19:36.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:36.873410 8698 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:19:36.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:36.874606 8664 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
05:19:36.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:36.874945 8664 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-1/data,/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-1/wal
05:19:36.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a5a0acd5ec4048e38717267b9d89fdfb"
05:19:36.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 05:19:30 on dist-test-slave-lhv2"
05:19:36.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:36.877569 8664 fs_report.cc:383] FS layout report
05:19:36.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:19:36.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-1/wal
05:19:36.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-1/wal
05:19:36.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-1/data/data
05:19:36.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:19:36.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:19:36.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:19:36.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:19:36.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:19:36.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:19:36.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:19:36.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:19:36.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:19:36.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:19:36.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:19:36.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:36.919031 8664 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:19:36.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:36.925395 8664 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:19:36.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:36.925940 8664 kserver.cc:163] Server-wide thread pool size limit: 3276
05:19:37.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:37.014707 8664 rpc_server.cc:307] RPC server started. Bound to: 127.8.27.253:44545
05:19:37.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:37.014859 8749 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.8.27.253:44545 every 8 connection(s)
05:19:37.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:37.018467 8664 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-1/data/info.pb
05:19:37.021 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:19:37.021 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.8.27.254:34253
05:19:37.021 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
05:19:37.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:37.019840 8303 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskHSJIN2/build/tsan/bin/kudu as pid 8664
05:19:37.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:37.023731 8303 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskHSJIN2/build/tsan/bin/kudu
05:19:37.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskHSJIN2/build/tsan/bin/kudu
05:19:37.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-0/wal
05:19:37.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-0/data
05:19:37.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:19:37.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:19:37.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:19:37.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:19:37.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:19:37.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:19:37.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-0/logs
05:19:37.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-0/data/info.pb
05:19:37.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:19:37.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:19:37.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:19:37.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:19:37.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:19:37.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:19:37.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:19:37.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:19:37.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:19:37.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:19:37.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:19:37.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:19:37.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.27.254:34253
05:19:37.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.27.254
05:19:37.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35839
05:19:37.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/krb5kdc/kudu_127.8.27.254.keytab
05:19:37.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.27.254
05:19:37.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:19:37.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
05:19:37.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
05:19:37.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.27.212:40337
05:19:37.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:19:37.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:19:37.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:19:37.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:19:37.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.27.254:34253,127.8.27.253:44545,127.8.27.252:39249
05:19:37.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:19:37.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37289/jwks.json
05:19:37.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
05:19:37.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:37.063310 8750 sys_catalog.cc:263] Verifying existing consensus state
05:19:37.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260928 05:19:37.064253 8750 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
05:19:37.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:37.064776 8664 master.cc:537] Master@127.8.27.253:44545 shutting down...
05:19:37.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:37.136953 8664 master.cc:552] Master@127.8.27.253:44545 shutdown complete.
05:19:37.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
05:19:37.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:37.186771 8677 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:19:37.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:37.187062 8677 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:19:37.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:37.372895 8751 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:19:37.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:37.373438 8751 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:19:37.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:37.374007 8751 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:19:37.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:37.424198 8751 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
05:19:37.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:37.424432 8751 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:19:37.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:37.424686 8751 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:19:37.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:37.424878 8751 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:19:37.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:37.425073 8751 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:19:37.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:37.425256 8751 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:19:37.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:37.425393 8751 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37289/jwks.json
05:19:37.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:37.463410 8751 master_runner.cc:386] Master server non-default flags:
05:19:37.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:19:37.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.27.212:40337
05:19:37.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:19:37.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:19:37.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-0/data
05:19:37.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-0/wal
05:19:37.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
05:19:37.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:19:37.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.27.254:34253,127.8.27.253:44545,127.8.27.252:39249
05:19:37.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/krb5kdc/kudu_127.8.27.254.keytab
05:19:37.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.27.254
05:19:37.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:19:37.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:19:37.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:19:37.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.27.254:34253
05:19:37.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:19:37.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:19:37.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:19:37.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37289/jwks.json
05:19:37.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:19:37.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
05:19:37.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
05:19:37.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:19:37.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:19:37.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-0/data/info.pb
05:19:37.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.27.254
05:19:37.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35839
05:19:37.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:19:37.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:19:37.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:19:37.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:19:37.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:19:37.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-0/logs
05:19:37.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:19:37.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:19:37.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:19:37.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:19:37.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:19:37.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:19:37.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:19:37.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 04:35:01 UTC on ccycloud.quasar-alllgi.root.comops.site
05:19:37.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676563
05:19:37.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:19:37.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:37.465173 8751 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:19:37.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:37.466383 8751 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:19:37.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:37.485554 8762 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:19:37.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:37.738231 8765 raft_consensus.cc:482] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
05:19:37.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:37.738700 8765 raft_consensus.cc:504] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "068ae5f7ad434c4291c050390e0b2580" member_type: VOTER last_known_addr { host: "127.8.27.254" port: 34253 } } peers { permanent_uuid: "a5a0acd5ec4048e38717267b9d89fdfb" member_type: VOTER last_known_addr { host: "127.8.27.253" port: 44545 } } peers { permanent_uuid: "5f42df00f0b34da18c6f5659ce854d38" member_type: VOTER last_known_addr { host: "127.8.27.252" port: 39249 } }
05:19:37.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:37.748764 8765 leader_election.cc:288] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 068ae5f7ad434c4291c050390e0b2580 (127.8.27.254:34253), a5a0acd5ec4048e38717267b9d89fdfb (127.8.27.253:44545)
05:19:37.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:37.752875 8614 leader_election.cc:334] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 068ae5f7ad434c4291c050390e0b2580 (127.8.27.254:34253): Network error: Client connection negotiation failed: client connection to 127.8.27.254:34253: connect: Connection refused (error 111)
05:19:37.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:37.754812 8613 leader_election.cc:334] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a5a0acd5ec4048e38717267b9d89fdfb (127.8.27.253:44545): Network error: Client connection negotiation failed: client connection to 127.8.27.253:44545: connect: Connection refused (error 111)
05:19:37.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:37.755229 8613 leader_election.cc:302] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5f42df00f0b34da18c6f5659ce854d38; no voters: 068ae5f7ad434c4291c050390e0b2580, a5a0acd5ec4048e38717267b9d89fdfb
05:19:37.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:37.756050 8765 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
05:19:38.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:38.189848 8677 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:19:38.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:38.190099 8677 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:19:38.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:37.485546 8760 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:19:38.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:37.487250 8751 server_base.cc:1034] running on GCE node
05:19:38.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:37.485807 8759 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:19:38.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:38.759265 8751 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:19:38.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:38.761988 8751 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:19:38.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:38.763335 8751 hybrid_clock.cc:648] HybridClock initialized: now 1790572778763298 us; error 50 us; skew 500 ppm
05:19:38.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:38.764124 8751 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:19:38.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 05:19:38 dist-test-slave-lhv2 krb5kdc[8315](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790572778, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.27.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:19:38.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:38.771462 8751 init.cc:377] Logged in from keytab as kudu/127.8.27.254@KRBTEST.COM (short username kudu)
05:19:38.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:38.778955 8751 webserver.cc:460] Webserver started at http://127.8.27.254:35839/ using document root <none> and password file <none>
05:19:38.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:38.779966 8751 fs_manager.cc:359] Metadata directory not provided
05:19:38.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:38.780230 8751 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:19:38.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:38.788357 8751 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.007s sys 0.001s
05:19:38.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:38.793459 8774 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:19:38.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:38.794660 8751 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
05:19:38.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:38.794958 8751 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-0/data,/tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-0/wal
05:19:38.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "068ae5f7ad434c4291c050390e0b2580"
05:19:38.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 05:19:28 on dist-test-slave-lhv2"
05:19:38.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:38.797351 8751 fs_report.cc:383] FS layout report
05:19:38.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:19:38.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-0/wal
05:19:38.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-0/wal
05:19:38.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-0/data/data
05:19:38.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:19:38.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:19:38.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:19:38.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:19:38.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:19:38.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:19:38.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:19:38.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:19:38.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:19:38.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:19:38.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:19:38.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:38.833483 8751 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:19:38.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:38.840204 8751 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:19:38.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:38.840838 8751 kserver.cc:163] Server-wide thread pool size limit: 3276
05:19:38.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:38.915648 8751 rpc_server.cc:307] RPC server started. Bound to: 127.8.27.254:34253
05:19:38.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:38.915768 8825 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.8.27.254:34253 every 8 connection(s)
05:19:38.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:38.918807 8751 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-0/data/info.pb
05:19:38.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:38.928045 8303 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskHSJIN2/build/tsan/bin/kudu as pid 8751
05:19:38.928 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:19:38.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:38.928755 8826 sys_catalog.cc:263] Verifying existing consensus state
05:19:38.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260928 05:19:38.929659 8826 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
05:19:38.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:38.930132 8751 master.cc:537] Master@127.8.27.254:34253 shutting down...
05:19:38.985 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.8.27.252:39249(127.8.27.252:39249)] connecting to peer
05:19:39.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:39.015144 8751 master.cc:552] Master@127.8.27.254:34253 shutdown complete.
05:19:39.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskHSJIN2/test-tmp/mini-kudu-cluster1553612750168749583/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
05:19:39.055 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.8.27.253:44545(127.8.27.253:44545)] connecting to peer
05:19:39.059 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.8.27.254:34253(127.8.27.254:34253)] connecting to peer
05:19:39.130 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.8.27.252:39249(127.8.27.252:39249)] Successfully connected to peer
05:19:39.135 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:19:39.135 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.27.253:44545(127.8.27.253:44545)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:19:39.137 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.27.253:44545: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:19:39.186 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:19:39.186 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.27.254:34253(127.8.27.254:34253)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:19:39.187 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.27.254:34253: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:19:39.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:39.193084 8677 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:19:39.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:39.193368 8677 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:19:39.202 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
05:19:39.210 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.8.27.252
05:19:39.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 05:19:39 dist-test-slave-lhv2 krb5kdc[8315](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790572773, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.8.27.252@KRBTEST.COM
05:19:39.429 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xb3d548fc, L:/127.0.0.1:44910 - R:/127.8.27.252:39249] using SASL/GSSAPI
05:19:39.445 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.27.252:39249,127.8.27.253:44545,127.8.27.254:34253; will retry
05:19:39.457 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.27.253:44545(127.8.27.253:44545)] connecting to peer
05:19:39.460 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:19:39.460 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.27.253:44545(127.8.27.253:44545)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:19:39.461 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.27.253:44545: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:19:39.462 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.27.254:34253(127.8.27.254:34253)] connecting to peer
05:19:39.464 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:19:39.464 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.27.254:34253(127.8.27.254:34253)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:19:39.465 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.27.254:34253: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:19:39.465 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.27.252:39249,127.8.27.253:44545,127.8.27.254:34253; will retry
05:19:39.475 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.27.253:44545(127.8.27.253:44545)] connecting to peer
05:19:39.478 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:19:39.478 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.27.253:44545(127.8.27.253:44545)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:19:39.480 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.27.253:44545: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:19:39.480 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.27.254:34253(127.8.27.254:34253)] connecting to peer
05:19:39.482 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:19:39.482 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.27.254:34253(127.8.27.254:34253)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:19:39.483 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.27.254:34253: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:19:39.484 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.27.252:39249,127.8.27.253:44545,127.8.27.254:34253; will retry
05:19:39.495 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.27.253:44545(127.8.27.253:44545)] connecting to peer
05:19:39.496 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.27.254:34253(127.8.27.254:34253)] connecting to peer
05:19:39.498 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:19:39.499 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.27.253:44545(127.8.27.253:44545)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:19:39.499 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:19:39.499 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.27.254:34253(127.8.27.254:34253)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:19:39.499 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.27.253:44545: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:19:39.500 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.27.254:34253: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:19:39.502 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.27.252:39249,127.8.27.253:44545,127.8.27.254:34253; will retry
05:19:39.515 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.27.253:44545(127.8.27.253:44545)] connecting to peer
05:19:39.516 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.27.254:34253(127.8.27.254:34253)] connecting to peer
05:19:39.517 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:19:39.518 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:19:39.518 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.27.253:44545(127.8.27.253:44545)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:19:39.518 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.27.254:34253(127.8.27.254:34253)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:19:39.519 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.27.254:34253: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:19:39.519 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.27.253:44545: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:19:39.521 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.27.252:39249,127.8.27.253:44545,127.8.27.254:34253; will retry
05:19:39.536 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.27.253:44545(127.8.27.253:44545)] connecting to peer
05:19:39.538 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.27.254:34253(127.8.27.254:34253)] connecting to peer
05:19:39.540 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:19:39.540 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.27.253:44545(127.8.27.253:44545)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:19:39.542 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.27.253:44545: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:19:39.545 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:19:39.545 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.27.254:34253(127.8.27.254:34253)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:19:39.546 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.27.254:34253: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:19:39.546 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.27.252:39249,127.8.27.253:44545,127.8.27.254:34253; will retry
05:19:39.596 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.27.253:44545(127.8.27.253:44545)] connecting to peer
05:19:39.597 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.27.254:34253(127.8.27.254:34253)] connecting to peer
05:19:39.598 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:19:39.598 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.27.253:44545(127.8.27.253:44545)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:19:39.600 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.27.253:44545: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:19:39.601 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:19:39.601 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.27.254:34253(127.8.27.254:34253)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:19:39.602 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.27.254:34253: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:19:39.602 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.27.252:39249,127.8.27.253:44545,127.8.27.254:34253; will retry
05:19:39.636 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.27.253:44545(127.8.27.253:44545)] connecting to peer
05:19:39.637 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.27.254:34253(127.8.27.254:34253)] connecting to peer
05:19:39.638 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:19:39.638 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.27.253:44545(127.8.27.253:44545)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:19:39.639 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.27.253:44545: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:19:39.641 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:19:39.642 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.27.254:34253(127.8.27.254:34253)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:19:39.642 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.27.254:34253: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:19:39.643 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.27.252:39249,127.8.27.253:44545,127.8.27.254:34253; will retry
05:19:39.655 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.27.253:44545(127.8.27.253:44545)] connecting to peer
05:19:39.656 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.27.254:34253(127.8.27.254:34253)] connecting to peer
05:19:39.657 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:19:39.657 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.27.253:44545(127.8.27.253:44545)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:19:39.659 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.27.253:44545: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:19:39.660 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:19:39.660 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.27.254:34253(127.8.27.254:34253)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:19:39.661 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.27.254:34253: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:19:39.661 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.27.252:39249,127.8.27.253:44545,127.8.27.254:34253; will retry
05:19:39.935 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.27.253:44545(127.8.27.253:44545)] connecting to peer
05:19:39.937 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.27.254:34253(127.8.27.254:34253)] connecting to peer
05:19:39.939 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:19:39.939 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.27.253:44545(127.8.27.253:44545)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:19:39.940 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.27.253:44545: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:19:39.940 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:19:39.940 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.27.254:34253(127.8.27.254:34253)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:19:39.941 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.27.254:34253: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:19:39.941 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.27.252:39249,127.8.27.253:44545,127.8.27.254:34253; will retry
05:19:40.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:40.196611 8677 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:19:40.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:40.196964 8677 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:19:40.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:40.219138 8592 master_runner.cc:186] Error getting master registration for 127.8.27.254:34253: Network error: Client connection negotiation failed: client connection to 127.8.27.254:34253: connect: Connection refused (error 111),
05:19:40.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:40.219552 8592 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:19:40.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:40.396970 8835 raft_consensus.cc:482] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
05:19:40.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:40.397452 8835 raft_consensus.cc:504] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "068ae5f7ad434c4291c050390e0b2580" member_type: VOTER last_known_addr { host: "127.8.27.254" port: 34253 } } peers { permanent_uuid: "a5a0acd5ec4048e38717267b9d89fdfb" member_type: VOTER last_known_addr { host: "127.8.27.253" port: 44545 } } peers { permanent_uuid: "5f42df00f0b34da18c6f5659ce854d38" member_type: VOTER last_known_addr { host: "127.8.27.252" port: 39249 } }
05:19:40.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:40.398996 8835 leader_election.cc:288] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 068ae5f7ad434c4291c050390e0b2580 (127.8.27.254:34253), a5a0acd5ec4048e38717267b9d89fdfb (127.8.27.253:44545)
05:19:40.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:40.403299 8614 leader_election.cc:334] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 068ae5f7ad434c4291c050390e0b2580 (127.8.27.254:34253): Network error: Client connection negotiation failed: client connection to 127.8.27.254:34253: connect: Connection refused (error 111)
05:19:40.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:40.403716 8613 leader_election.cc:334] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a5a0acd5ec4048e38717267b9d89fdfb (127.8.27.253:44545): Network error: Client connection negotiation failed: client connection to 127.8.27.253:44545: connect: Connection refused (error 111)
05:19:40.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:40.404054 8613 leader_election.cc:302] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5f42df00f0b34da18c6f5659ce854d38; no voters: 068ae5f7ad434c4291c050390e0b2580, a5a0acd5ec4048e38717267b9d89fdfb
05:19:40.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:40.404595 8835 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
05:19:40.575 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.27.253:44545(127.8.27.253:44545)] connecting to peer
05:19:40.575 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.27.254:34253(127.8.27.254:34253)] connecting to peer
05:19:40.578 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:19:40.578 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.27.253:44545(127.8.27.253:44545)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:19:40.579 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.27.253:44545: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:19:40.579 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:19:40.579 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.27.254:34253(127.8.27.254:34253)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:19:40.579 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.27.254:34253: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:19:40.579 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.27.252:39249,127.8.27.253:44545,127.8.27.254:34253; will retry
05:19:41.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:41.199815 8677 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:19:41.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:41.200160 8677 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:19:41.935 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.27.253:44545(127.8.27.253:44545)] connecting to peer
05:19:41.936 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.27.254:34253(127.8.27.254:34253)] connecting to peer
05:19:41.938 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:19:41.939 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.27.253:44545(127.8.27.253:44545)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:19:41.939 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.27.253:44545: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:19:41.940 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:19:41.940 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.27.254:34253(127.8.27.254:34253)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:19:41.940 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.27.254:34253: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:19:41.940 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.27.252:39249,127.8.27.253:44545,127.8.27.254:34253; will retry
05:19:42.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:42.203084 8677 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:19:42.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:42.203382 8677 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:19:42.235 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.27.253:44545(127.8.27.253:44545)] connecting to peer
05:19:42.236 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.27.254:34253(127.8.27.254:34253)] connecting to peer
05:19:42.237 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:19:42.238 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.27.253:44545(127.8.27.253:44545)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:19:42.238 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.27.253:44545: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:19:42.238 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:19:42.239 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.27.254:34253(127.8.27.254:34253)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:19:42.239 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.27.254:34253: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:19:42.240 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.27.252:39249,127.8.27.253:44545,127.8.27.254:34253; will retry
05:19:42.475 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.27.253:44545(127.8.27.253:44545)] connecting to peer
05:19:42.476 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.27.254:34253(127.8.27.254:34253)] connecting to peer
05:19:42.477 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:19:42.478 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.27.253:44545(127.8.27.253:44545)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:19:42.478 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.27.253:44545: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:19:42.479 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:19:42.479 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.27.254:34253(127.8.27.254:34253)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:19:42.479 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.27.254:34253: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:19:42.479 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.27.252:39249,127.8.27.253:44545,127.8.27.254:34253; will retry
05:19:42.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:42.617714 8842 raft_consensus.cc:482] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
05:19:42.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:42.618096 8842 raft_consensus.cc:504] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "068ae5f7ad434c4291c050390e0b2580" member_type: VOTER last_known_addr { host: "127.8.27.254" port: 34253 } } peers { permanent_uuid: "a5a0acd5ec4048e38717267b9d89fdfb" member_type: VOTER last_known_addr { host: "127.8.27.253" port: 44545 } } peers { permanent_uuid: "5f42df00f0b34da18c6f5659ce854d38" member_type: VOTER last_known_addr { host: "127.8.27.252" port: 39249 } }
05:19:42.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:42.619786 8842 leader_election.cc:288] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 068ae5f7ad434c4291c050390e0b2580 (127.8.27.254:34253), a5a0acd5ec4048e38717267b9d89fdfb (127.8.27.253:44545)
05:19:42.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:42.621670 8614 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.27.254:34253: connect: Connection refused (error 111) [suppressed 5 similar messages]
05:19:42.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:42.624442 8613 leader_election.cc:334] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a5a0acd5ec4048e38717267b9d89fdfb (127.8.27.253:44545): Network error: Client connection negotiation failed: client connection to 127.8.27.253:44545: connect: Connection refused (error 111)
05:19:42.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:42.625056 8614 leader_election.cc:334] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 068ae5f7ad434c4291c050390e0b2580 (127.8.27.254:34253): Network error: Client connection negotiation failed: client connection to 127.8.27.254:34253: connect: Connection refused (error 111)
05:19:42.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:42.625495 8614 leader_election.cc:302] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5f42df00f0b34da18c6f5659ce854d38; no voters: 068ae5f7ad434c4291c050390e0b2580, a5a0acd5ec4048e38717267b9d89fdfb
05:19:42.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:42.626195 8842 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
05:19:43.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:43.206424 8677 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:19:43.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:43.206715 8677 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:19:44.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:44.210225 8677 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:19:44.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:44.210551 8677 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:19:44.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 05:19:44 dist-test-slave-lhv2 krb5kdc[8315](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790572784, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.27.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:19:44.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:44.882419 8606 init.cc:295] Successfully reacquired a new kerberos TGT
05:19:45.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:45.213559 8677 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:19:45.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:45.213837 8677 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:19:45.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:45.222788 8592 master_runner.cc:186] Error getting master registration for 127.8.27.254:34253: Network error: Client connection negotiation failed: client connection to 127.8.27.254:34253: connect: Connection refused (error 111),
05:19:45.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:45.223291 8592 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:19:45.515 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.27.253:44545(127.8.27.253:44545)] connecting to peer
05:19:45.517 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.27.254:34253(127.8.27.254:34253)] connecting to peer
05:19:45.518 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:19:45.519 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.27.253:44545(127.8.27.253:44545)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:19:45.519 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.27.253:44545: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:19:45.520 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:19:45.520 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.27.254:34253(127.8.27.254:34253)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:19:45.520 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.27.254:34253: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:19:45.521 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.27.252:39249,127.8.27.253:44545,127.8.27.254:34253; will retry
05:19:46.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:46.216640 8677 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:19:46.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:46.216943 8677 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:19:46.835 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.27.253:44545(127.8.27.253:44545)] connecting to peer
05:19:46.837 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.27.254:34253(127.8.27.254:34253)] connecting to peer
05:19:46.837 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:19:46.838 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.27.253:44545(127.8.27.253:44545)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:19:46.838 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.27.253:44545: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:19:46.841 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:19:46.841 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.27.254:34253(127.8.27.254:34253)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:19:46.841 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.27.254:34253: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:19:46.841 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.27.252:39249,127.8.27.253:44545,127.8.27.254:34253; will retry
05:19:47.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:47.220091 8677 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:19:47.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:47.220381 8677 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:19:48.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:48.223109 8677 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:19:48.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:48.223457 8677 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:19:49.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:49.226336 8677 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:19:49.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:49.226605 8677 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:19:49.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:49.452561 8853 raft_consensus.cc:482] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
05:19:49.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:49.453099 8853 raft_consensus.cc:504] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "068ae5f7ad434c4291c050390e0b2580" member_type: VOTER last_known_addr { host: "127.8.27.254" port: 34253 } } peers { permanent_uuid: "a5a0acd5ec4048e38717267b9d89fdfb" member_type: VOTER last_known_addr { host: "127.8.27.253" port: 44545 } } peers { permanent_uuid: "5f42df00f0b34da18c6f5659ce854d38" member_type: VOTER last_known_addr { host: "127.8.27.252" port: 39249 } }
05:19:49.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:49.454941 8853 leader_election.cc:288] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 068ae5f7ad434c4291c050390e0b2580 (127.8.27.254:34253), a5a0acd5ec4048e38717267b9d89fdfb (127.8.27.253:44545)
05:19:49.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:49.457078 8614 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.27.254:34253: connect: Connection refused (error 111) [suppressed 1 similar messages]
05:19:49.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:49.460268 8613 leader_election.cc:334] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a5a0acd5ec4048e38717267b9d89fdfb (127.8.27.253:44545): Network error: Client connection negotiation failed: client connection to 127.8.27.253:44545: connect: Connection refused (error 111)
05:19:49.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:49.461254 8614 leader_election.cc:334] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 068ae5f7ad434c4291c050390e0b2580 (127.8.27.254:34253): Network error: Client connection negotiation failed: client connection to 127.8.27.254:34253: connect: Connection refused (error 111)
05:19:49.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:49.461557 8614 leader_election.cc:302] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5f42df00f0b34da18c6f5659ce854d38; no voters: 068ae5f7ad434c4291c050390e0b2580, a5a0acd5ec4048e38717267b9d89fdfb
05:19:49.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:49.462144 8853 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
05:19:50.216 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.27.253:44545(127.8.27.253:44545)] connecting to peer
05:19:50.218 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.27.254:34253(127.8.27.254:34253)] connecting to peer
05:19:50.218 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:19:50.218 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.27.253:44545(127.8.27.253:44545)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:19:50.219 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.27.253:44545: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:19:50.220 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:19:50.220 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.27.254:34253(127.8.27.254:34253)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:19:50.220 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.27.254:34253: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:19:50.221 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.27.252:39249,127.8.27.253:44545,127.8.27.254:34253; will retry
05:19:50.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:50.225876 8592 master_runner.cc:186] Error getting master registration for 127.8.27.254:34253: Network error: Client connection negotiation failed: client connection to 127.8.27.254:34253: connect: Connection refused (error 111),
05:19:50.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:50.226251 8592 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:19:50.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:50.229310 8677 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:19:50.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:50.229542 8677 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:19:50.475 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.27.253:44545(127.8.27.253:44545)] connecting to peer
05:19:50.477 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.27.254:34253(127.8.27.254:34253)] connecting to peer
05:19:50.477 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:19:50.478 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.27.253:44545(127.8.27.253:44545)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:19:50.478 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.27.253:44545: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:19:50.479 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:19:50.479 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.27.254:34253(127.8.27.254:34253)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:19:50.480 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.27.254:34253: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:19:50.480 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.27.252:39249,127.8.27.253:44545,127.8.27.254:34253; will retry
05:19:51.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:51.232273 8677 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:19:51.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:51.232532 8677 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:19:52.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:52.235247 8677 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:19:52.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:52.235500 8677 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:19:52.715 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.27.253:44545(127.8.27.253:44545)] connecting to peer
05:19:52.717 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.27.254:34253(127.8.27.254:34253)] connecting to peer
05:19:52.718 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:19:52.721 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.27.253:44545(127.8.27.253:44545)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:19:52.721 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.27.253:44545: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:19:52.722 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:19:52.722 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.27.254:34253(127.8.27.254:34253)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:19:52.722 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.27.254:34253: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:19:52.722 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.27.252:39249,127.8.27.253:44545,127.8.27.254:34253; will retry
05:19:53.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:53.240387 8677 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:19:53.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:53.240669 8677 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:19:53.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:53.868777 8864 raft_consensus.cc:482] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
05:19:53.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:53.869483 8864 raft_consensus.cc:504] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "068ae5f7ad434c4291c050390e0b2580" member_type: VOTER last_known_addr { host: "127.8.27.254" port: 34253 } } peers { permanent_uuid: "a5a0acd5ec4048e38717267b9d89fdfb" member_type: VOTER last_known_addr { host: "127.8.27.253" port: 44545 } } peers { permanent_uuid: "5f42df00f0b34da18c6f5659ce854d38" member_type: VOTER last_known_addr { host: "127.8.27.252" port: 39249 } }
05:19:53.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:53.871531 8864 leader_election.cc:288] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 068ae5f7ad434c4291c050390e0b2580 (127.8.27.254:34253), a5a0acd5ec4048e38717267b9d89fdfb (127.8.27.253:44545)
05:19:53.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:53.877024 8613 leader_election.cc:334] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a5a0acd5ec4048e38717267b9d89fdfb (127.8.27.253:44545): Network error: Client connection negotiation failed: client connection to 127.8.27.253:44545: connect: Connection refused (error 111)
05:19:53.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:53.877782 8614 leader_election.cc:334] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 068ae5f7ad434c4291c050390e0b2580 (127.8.27.254:34253): Network error: Client connection negotiation failed: client connection to 127.8.27.254:34253: connect: Connection refused (error 111)
05:19:53.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:53.878156 8614 leader_election.cc:302] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5f42df00f0b34da18c6f5659ce854d38; no voters: 068ae5f7ad434c4291c050390e0b2580, a5a0acd5ec4048e38717267b9d89fdfb
05:19:53.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:53.878852 8864 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
05:19:54.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:54.244179 8677 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:19:54.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:54.244458 8677 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:19:54.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 05:19:54 dist-test-slave-lhv2 krb5kdc[8315](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790572794, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.27.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:19:54.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:54.885066 8606 init.cc:295] Successfully reacquired a new kerberos TGT
05:19:55.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:55.229242 8592 master_runner.cc:186] Error getting master registration for 127.8.27.254:34253: Network error: Client connection negotiation failed: client connection to 127.8.27.254:34253: connect: Connection refused (error 111),
05:19:55.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:19:55.229676 8592 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:19:55.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:55.247253 8677 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:19:55.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:55.247501 8677 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:19:56.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:56.250358 8677 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:19:56.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:56.250658 8677 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:19:56.755 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.27.253:44545(127.8.27.253:44545)] connecting to peer
05:19:56.758 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:19:56.758 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.27.253:44545(127.8.27.253:44545)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:19:56.758 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.27.253:44545: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:19:56.758 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.27.254:34253(127.8.27.254:34253)] connecting to peer
05:19:56.760 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:19:56.760 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.27.254:34253(127.8.27.254:34253)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:19:56.760 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.27.254:34253: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:19:56.761 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.27.252:39249,127.8.27.253:44545,127.8.27.254:34253; will retry
05:19:57.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:57.253535 8677 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:19:57.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:57.253822 8677 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:19:58.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:58.156033 7727 debug-util.cc:398] Leaking SignalData structure 0x7b08000b7b20 after lost signal to thread 7662
05:19:58.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:58.156852 7727 debug-util.cc:398] Leaking SignalData structure 0x7b08000bcf60 after lost signal to thread 7730
05:19:58.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:58.256861 8677 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:19:58.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:58.257113 8677 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:19:59.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:59.260296 8677 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:19:59.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:19:59.260587 8677 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:19:59.755 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.27.253:44545(127.8.27.253:44545)] connecting to peer
05:19:59.756 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.27.254:34253(127.8.27.254:34253)] connecting to peer
05:19:59.757 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:19:59.757 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.27.253:44545(127.8.27.253:44545)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:19:59.757 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:19:59.757 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.27.253:44545: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:19:59.757 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.27.254:34253(127.8.27.254:34253)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:19:59.757 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.27.254:34253: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:19:59.759 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.27.252:39249,127.8.27.253:44545,127.8.27.254:34253; will retry
05:20:00.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:20:00.232345 8592 master_runner.cc:186] Error getting master registration for 127.8.27.254:34253: Network error: Client connection negotiation failed: client connection to 127.8.27.254:34253: connect: Connection refused (error 111),
05:20:00.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:20:00.232857 8592 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:20:00.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:20:00.263720 8677 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:20:00.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:20:00.264014 8677 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:20:00.715 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.27.253:44545(127.8.27.253:44545)] connecting to peer
05:20:00.716 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.27.254:34253(127.8.27.254:34253)] connecting to peer
05:20:00.717 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:20:00.717 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.27.253:44545(127.8.27.253:44545)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:20:00.717 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.27.253:44545: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:20:00.717 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:20:00.717 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.27.254:34253(127.8.27.254:34253)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:20:00.717 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.27.254:34253: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:20:00.719 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.27.252:39249,127.8.27.253:44545,127.8.27.254:34253; will retry
05:20:01.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:20:01.266791 8677 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:20:01.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:20:01.267035 8677 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:20:02.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:20:02.270229 8677 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:20:02.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:20:02.270463 8677 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:20:02.375 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.27.253:44545(127.8.27.253:44545)] connecting to peer
05:20:02.377 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.27.254:34253(127.8.27.254:34253)] connecting to peer
05:20:02.377 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:20:02.377 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.27.253:44545(127.8.27.253:44545)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:20:02.378 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.27.253:44545: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:20:02.378 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:20:02.378 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.27.254:34253(127.8.27.254:34253)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:20:02.378 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.27.254:34253: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:20:02.379 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.27.252:39249,127.8.27.253:44545,127.8.27.254:34253; will retry
05:20:03.075 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.27.253:44545(127.8.27.253:44545)] connecting to peer
05:20:03.076 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.27.254:34253(127.8.27.254:34253)] connecting to peer
05:20:03.076 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:20:03.076 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.27.253:44545(127.8.27.253:44545)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:20:03.077 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.27.253:44545: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:20:03.077 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:20:03.077 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.27.254:34253(127.8.27.254:34253)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:20:03.077 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.27.254:34253: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:20:03.079 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.27.252:39249,127.8.27.253:44545,127.8.27.254:34253; will retry
05:20:03.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:20:03.273317 8677 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:20:03.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:20:03.273561 8677 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:20:04.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:20:04.276984 8677 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:20:04.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:20:04.277395 8677 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:20:04.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 05:20:04 dist-test-slave-lhv2 krb5kdc[8315](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790572804, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.27.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:20:04.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:20:04.887565 8606 init.cc:295] Successfully reacquired a new kerberos TGT
05:20:05.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:20:05.236299 8592 master_runner.cc:186] Error getting master registration for 127.8.27.254:34253: Network error: Client connection negotiation failed: client connection to 127.8.27.254:34253: connect: Connection refused (error 111),
05:20:05.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:20:05.236675 8592 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:20:05.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:20:05.280170 8677 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5f42df00f0b34da18c6f5659ce854d38: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:20:05.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 05:20:05.280438 8677 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:20:05.795 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.27.253:44545(127.8.27.253:44545)] connecting to peer
05:20:05.796 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.8.27.254:34253(127.8.27.254:34253)] connecting to peer
05:20:05.796 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:20:05.796 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.27.253:44545(127.8.27.253:44545)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:20:05.797 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.8.27.253:44545: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545
05:20:05.797 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:20:05.797 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.27.254:34253(127.8.27.254:34253)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:20:05.797 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.8.27.254:34253: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
05:20:05.799 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.8.27.252:39249,127.8.27.253:44545,127.8.27.254:34253; will retry
05:20:05.804 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=26866), Traces: [0ms] refreshing cache from master, [33ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.252:39249, [115ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.253:44545, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.254:34253, [196ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.253:44545: Network error: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545, [246ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.254:34253: Network error: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [501ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.252:39249: OK, [506ms] delaying RPC due to: Service unavailable: Master config (127.8.27.252:39249,127.8.27.253:44545,127.8.27.254:34253) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [514ms] refreshing cache from master, [515ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.252:39249, [517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.253:44545, [519ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.254:34253, [520ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.253:44545: Network error: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545, [521ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.252:39249: OK, [524ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.254:34253: Network error: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [526ms] delaying RPC due to: Service unavailable: Master config (127.8.27.252:39249,127.8.27.253:44545,127.8.27.254:34253) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [535ms] refreshing cache from master, [535ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.252:39249, [535ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.253:44545, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.254:34253, [538ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.253:44545: Network error: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545, [540ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.252:39249: OK, [543ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.254:34253: Network error: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [544ms] delaying RPC due to: Service unavailable: Master config (127.8.27.252:39249,127.8.27.253:44545,127.8.27.254:34253) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [554ms] refreshing cache from master, [554ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.252:39249, [555ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.253:44545, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.254:34253, [559ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.253:44545: Network error: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545, [559ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.254:34253: Network error: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [561ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.252:39249: OK, [562ms] delaying RPC due to: Service unavailable: Master config (127.8.27.252:39249,127.8.27.253:44545,127.8.27.254:34253) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [574ms] refreshing cache from master, [574ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.252:39249, [574ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.253:44545, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.254:34253, [578ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.253:44545: Network error: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545, [578ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.254:34253: Network error: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [580ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.252:39249: OK, [581ms] delaying RPC due to: Service unavailable: Master config (127.8.27.252:39249,127.8.27.253:44545,127.8.27.254:34253) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545, [595ms] refreshing cache from master, [595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.252:39249, [595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.253:44545, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.254:34253, [600ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.253:44545: Network error: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545, [604ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.252:39249: OK, [605ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.254:34253: Network error: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [606ms] delaying RPC due to: Service unavailable: Master config (127.8.27.252:39249,127.8.27.253:44545,127.8.27.254:34253) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [655ms] refreshing cache from master, [655ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.252:39249, [655ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.253:44545, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.254:34253, [659ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.252:39249: OK, [659ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.253:44545: Network error: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545, [661ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.254:34253: Network error: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [663ms] delaying RPC due to: Service unavailable: Master config (127.8.27.252:39249,127.8.27.253:44545,127.8.27.254:34253) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [695ms] refreshing cache from master, [695ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.252:39249, [695ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.253:44545, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.254:34253, [698ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.253:44545: Network error: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545, [700ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.252:39249: OK, [702ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.254:34253: Network error: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [703ms] delaying RPC due to: Service unavailable: Master config (127.8.27.252:39249,127.8.27.253:44545,127.8.27.254:34253) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [714ms] refreshing cache from master, [714ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.252:39249, [715ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.253:44545, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.254:34253, [718ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.253:44545: Network error: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545, [719ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.252:39249: OK, [720ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.254:34253: Network error: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [721ms] delaying RPC due to: Service unavailable: Master config (127.8.27.252:39249,127.8.27.253:44545,127.8.27.254:34253) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [995ms] refreshing cache from master, [995ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.252:39249, [995ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.253:44545, [996ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.254:34253, [999ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.253:44545: Network error: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545, [999ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.252:39249: OK, [1000ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.254:34253: Network error: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [1001ms] delaying RPC due to: Service unavailable: Master config (127.8.27.252:39249,127.8.27.253:44545,127.8.27.254:34253) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [1634ms] refreshing cache from master, [1634ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.252:39249, [1635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.253:44545, [1635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.254:34253, [1638ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.253:44545: Network error: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545, [1639ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.252:39249: OK, [1639ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.254:34253: Network error: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [1639ms] delaying RPC due to: Service unavailable: Master config (127.8.27.252:39249,127.8.27.253:44545,127.8.27.254:34253) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [2994ms] refreshing cache from master, [2994ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.252:39249, [2995ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.253:44545, [2996ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.254:34253, [2999ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.253:44545: Network error: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545, [2999ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.252:39249: OK, [3000ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.254:34253: Network error: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [3000ms] delaying RPC due to: Service unavailable: Master config (127.8.27.252:39249,127.8.27.253:44545,127.8.27.254:34253) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [3294ms] refreshing cache from master, [3294ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.252:39249, [3295ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.253:44545, [3296ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.254:34253, [3298ms] too many traces: truncated at 100 traces, deferred=Deferred@595595211(state=PENDING, result=null, callback=(continuation of Deferred@1254517119 after retry RPC after error@109429248) -> (continuation of Deferred@895107142 after retry RPC after error@378007409) -> (continuation of Deferred@1878257999 after retry RPC after error@1204730739) -> (continuation of Deferred@544316348 after retry RPC after error@161748132) -> (continuation of Deferred@2041443670 after retry RPC after error@1267391539) -> (continuation of Deferred@1728713349 after retry RPC after error@1319255028) -> (continuation of Deferred@1479766067 after retry RPC after error@1170555970) -> (continuation of Deferred@1928128581 after retry RPC after error@768479351) -> (continuation of Deferred@1849646869 after retry RPC after error@28832196) -> (continuation of Deferred@344756087 after retry RPC after error@1226598457) -> (continuation of Deferred@1573863928 after retry RPC after error@1674097236) -> (continuation of Deferred@323972717 after retry RPC after error@960004509) -> (continuation of Deferred@509597635 after retry RPC after error@950246478) -> (continuation of Deferred@1350021425 after retry RPC after error@759097035) -> (continuation of Deferred@1296026976 after retry RPC after error@1775925605) -> (continuation of Deferred@80327510 after retry RPC after error@1578833338) -> (continuation of Deferred@1836877780 after retry RPC after error@1473759823) -> (continuation of Deferred@508395744 after retry RPC after error@410906710) -> (continuation of Deferred@297174698 after retry RPC after error@2019210081) -> (continuation of Deferred@174224906 after retry RPC after error@2112538704) -> (continuation of Deferred@1489801873 after retry RPC after error@1973482322) -> (continuation of Deferred@1818954186 after retry RPC after error@1206445151) -> (continuation of Deferred@2064428585 after retry RPC after error@155510905) -> (continuation of Deferred@1029014978 after retry RPC after error@1033847851), errback=(continuation of Deferred@1254517119 after retry RPC after error@109429248) -> (continuation of Deferred@895107142 after retry RPC after error@378007409) -> (continuation of Deferred@1878257999 after retry RPC after error@1204730739) -> (continuation of Deferred@544316348 after retry RPC after error@161748132) -> (continuation of Deferred@2041443670 after retry RPC after error@1267391539) -> (continuation of Deferred@1728713349 after retry RPC after error@1319255028) -> (continuation of Deferred@1479766067 after retry RPC after error@1170555970) -> (continuation of Deferred@1928128581 after retry RPC after error@768479351) -> (continuation of Deferred@1849646869 after retry RPC after error@28832196) -> (continuation of Deferred@344756087 after retry RPC after error@1226598457) -> (continuation of Deferred@1573863928 after retry RPC after error@1674097236) -> (continuation of Deferred@323972717 after retry RPC after error@960004509) -> (continuation of Deferred@509597635 after retry RPC after error@950246478) -> (continuation of Deferred@1350021425 after retry RPC after error@759097035) -> (continuation of Deferred@1296026976 after retry RPC after error@1775925605) -> (continuation of Deferred@80327510 after retry RPC after error@1578833338) -> (continuation of Deferred@1836877780 after retry RPC after error@1473759823) -> (continuation of Deferred@508395744 after retry RPC after error@410906710) -> (continuation of Deferred@297174698 after retry RPC after error@2019210081) -> (continuation of Deferred@174224906 after retry RPC after error@2112538704) -> (continuation of Deferred@1489801873 after retry RPC after error@1973482322) -> (continuation of Deferred@1818954186 after retry RPC after error@1206445151) -> (continuation of Deferred@2064428585 after retry RPC after error@155510905) -> (continuation of Deferred@1029014978 after retry RPC after error@1033847851)))
05:20:05.809 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.8.27.254:34253(127.8.27.254:34253)] disconnecting while in state TERMINATED
05:20:05.810 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.8.27.252:39249(127.8.27.252:39249)] disconnecting while in state READY
05:20:05.810 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.8.27.253:44545(127.8.27.253:44545)] disconnecting while in state TERMINATED
05:20:05.816 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
05:20:05.818 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
05:20:05.819 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.8.27.252:39249(127.8.27.252:39249)] handling channelInactive
05:20:05.824 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.27.252:39249(127.8.27.252:39249)] cleaning up while in state READY due to: connection closed
05:20:05.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:20:05.832760 8303 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskHSJIN2/build/tsan/bin/kudu with pid 8592
05:20:07.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-28T05:20:07Z chronyd exiting
05:20:08.165 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
05:20:08.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:20:08.166891 7576 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskHSJIN2/build/tsan/bin/kudu with pid 7856
05:20:08.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:20:08.192766 7576 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskHSJIN2/build/tsan/bin/kudu with pid 8018
05:20:08.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:20:08.223934 7576 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskHSJIN2/build/tsan/bin/kudu with pid 8162
05:20:08.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:20:08.266615 7576 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskHSJIN2/build/tsan/bin/kudu with pid 7588
05:20:08.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:20:08.296751 7576 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskHSJIN2/build/tsan/bin/kudu with pid 7658
05:20:08.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 05:20:08.327579 7576 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskHSJIN2/build/tsan/bin/kudu with pid 7733
05:20:09.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-28T05:20:09Z chronyd exiting
05:20:10.528 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=26866), Traces: [0ms] refreshing cache from master, [33ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.252:39249, [115ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.253:44545, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.254:34253, [196ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.253:44545: Network error: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545, [246ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.254:34253: Network error: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [501ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.252:39249: OK, [506ms] delaying RPC due to: Service unavailable: Master config (127.8.27.252:39249,127.8.27.253:44545,127.8.27.254:34253) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [514ms] refreshing cache from master, [515ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.252:39249, [517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.253:44545, [519ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.254:34253, [520ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.253:44545: Network error: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545, [521ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.252:39249: OK, [524ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.254:34253: Network error: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [526ms] delaying RPC due to: Service unavailable: Master config (127.8.27.252:39249,127.8.27.253:44545,127.8.27.254:34253) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [535ms] refreshing cache from master, [535ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.252:39249, [535ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.253:44545, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.254:34253, [538ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.253:44545: Network error: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545, [540ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.252:39249: OK, [543ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.254:34253: Network error: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [544ms] delaying RPC due to: Service unavailable: Master config (127.8.27.252:39249,127.8.27.253:44545,127.8.27.254:34253) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [554ms] refreshing cache from master, [554ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.252:39249, [555ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.253:44545, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.254:34253, [559ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.253:44545: Network error: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545, [559ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.254:34253: Network error: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [561ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.252:39249: OK, [562ms] delaying RPC due to: Service unavailable: Master config (127.8.27.252:39249,127.8.27.253:44545,127.8.27.254:34253) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [574ms] refreshing cache from master, [574ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.252:39249, [574ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.253:44545, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.254:34253, [578ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.253:44545: Network error: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545, [578ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.254:34253: Network error: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [580ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.252:39249: OK, [581ms] delaying RPC due to: Service unavailable: Master config (127.8.27.252:39249,127.8.27.253:44545,127.8.27.254:34253) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545, [595ms] refreshing cache from master, [595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.252:39249, [595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.253:44545, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.254:34253, [600ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.253:44545: Network error: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545, [604ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.252:39249: OK, [605ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.254:34253: Network error: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [606ms] delaying RPC due to: Service unavailable: Master config (127.8.27.252:39249,127.8.27.253:44545,127.8.27.254:34253) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [655ms] refreshing cache from master, [655ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.252:39249, [655ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.253:44545, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.254:34253, [659ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.252:39249: OK, [659ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.253:44545: Network error: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545, [661ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.254:34253: Network error: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [663ms] delaying RPC due to: Service unavailable: Master config (127.8.27.252:39249,127.8.27.253:44545,127.8.27.254:34253) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [695ms] refreshing cache from master, [695ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.252:39249, [695ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.253:44545, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.254:34253, [698ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.253:44545: Network error: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545, [700ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.252:39249: OK, [702ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.254:34253: Network error: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [703ms] delaying RPC due to: Service unavailable: Master config (127.8.27.252:39249,127.8.27.253:44545,127.8.27.254:34253) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [714ms] refreshing cache from master, [714ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.252:39249, [715ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.253:44545, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.254:34253, [718ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.253:44545: Network error: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545, [719ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.252:39249: OK, [720ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.254:34253: Network error: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [721ms] delaying RPC due to: Service unavailable: Master config (127.8.27.252:39249,127.8.27.253:44545,127.8.27.254:34253) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [995ms] refreshing cache from master, [995ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.252:39249, [995ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.253:44545, [996ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.254:34253, [999ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.253:44545: Network error: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545, [999ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.252:39249: OK, [1000ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.254:34253: Network error: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [1001ms] delaying RPC due to: Service unavailable: Master config (127.8.27.252:39249,127.8.27.253:44545,127.8.27.254:34253) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [1634ms] refreshing cache from master, [1634ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.252:39249, [1635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.253:44545, [1635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.254:34253, [1638ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.253:44545: Network error: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545, [1639ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.252:39249: OK, [1639ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.254:34253: Network error: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [1639ms] delaying RPC due to: Service unavailable: Master config (127.8.27.252:39249,127.8.27.253:44545,127.8.27.254:34253) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [2994ms] refreshing cache from master, [2994ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.252:39249, [2995ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.253:44545, [2996ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.254:34253, [2999ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.253:44545: Network error: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545, [2999ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.252:39249: OK, [3000ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.254:34253: Network error: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [3000ms] delaying RPC due to: Service unavailable: Master config (127.8.27.252:39249,127.8.27.253:44545,127.8.27.254:34253) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [3294ms] refreshing cache from master, [3294ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.252:39249, [3295ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.253:44545, [3296ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.254:34253, [3298ms] too many traces: truncated at 100 traces, deferred=Deferred@595595211(state=PENDING, result=null, callback=(continuation of Deferred@1254517119 after retry RPC after error@109429248) -> (continuation of Deferred@895107142 after retry RPC after error@378007409) -> (continuation of Deferred@1878257999 after retry RPC after error@1204730739) -> (continuation of Deferred@544316348 after retry RPC after error@161748132) -> (continuation of Deferred@2041443670 after retry RPC after error@1267391539) -> (continuation of Deferred@1728713349 after retry RPC after error@1319255028) -> (continuation of Deferred@1479766067 after retry RPC after error@1170555970) -> (continuation of Deferred@1928128581 after retry RPC after error@768479351) -> (continuation of Deferred@1849646869 after retry RPC after error@28832196) -> (continuation of Deferred@344756087 after retry RPC after error@1226598457) -> (continuation of Deferred@1573863928 after retry RPC after error@1674097236) -> (continuation of Deferred@323972717 after retry RPC after error@960004509) -> (continuation of Deferred@509597635 after retry RPC after error@950246478) -> (continuation of Deferred@1350021425 after retry RPC after error@759097035) -> (continuation of Deferred@1296026976 after retry RPC after error@1775925605) -> (continuation of Deferred@80327510 after retry RPC after error@1578833338) -> (continuation of Deferred@1836877780 after retry RPC after error@1473759823) -> (continuation of Deferred@508395744 after retry RPC after error@410906710) -> (continuation of Deferred@297174698 after retry RPC after error@2019210081) -> (continuation of Deferred@174224906 after retry RPC after error@2112538704) -> (continuation of Deferred@1489801873 after retry RPC after error@1973482322) -> (continuation of Deferred@1818954186 after retry RPC after error@1206445151) -> (continuation of Deferred@2064428585 after retry RPC after error@155510905) -> (continuation of Deferred@1029014978 after retry RPC after error@1033847851), errback=(continuation of Deferred@1254517119 after retry RPC after error@109429248) -> (continuation of Deferred@895107142 after retry RPC after error@378007409) -> (continuation of Deferred@1878257999 after retry RPC after error@1204730739) -> (continuation of Deferred@544316348 after retry RPC after error@161748132) -> (continuation of Deferred@2041443670 after retry RPC after error@1267391539) -> (continuation of Deferred@1728713349 after retry RPC after error@1319255028) -> (continuation of Deferred@1479766067 after retry RPC after error@1170555970) -> (continuation of Deferred@1928128581 after retry RPC after error@768479351) -> (continuation of Deferred@1849646869 after retry RPC after error@28832196) -> (continuation of Deferred@344756087 after retry RPC after error@1226598457) -> (continuation of Deferred@1573863928 after retry RPC after error@1674097236) -> (continuation of Deferred@323972717 after retry RPC after error@960004509) -> (continuation of Deferred@509597635 after retry RPC after error@950246478) -> (continuation of Deferred@1350021425 after retry RPC after error@759097035) -> (continuation of Deferred@1296026976 after retry RPC after error@1775925605) -> (continuation of Deferred@80327510 after retry RPC after error@1578833338) -> (continuation of Deferred@1836877780 after retry RPC after error@1473759823) -> (continuation of Deferred@508395744 after retry RPC after error@410906710) -> (continuation of Deferred@297174698 after retry RPC after error@2019210081) -> (continuation of Deferred@174224906 after retry RPC after error@2112538704) -> (continuation of Deferred@1489801873 after retry RPC after error@1973482322) -> (continuation of Deferred@1818954186 after retry RPC after error@1206445151) -> (continuation of Deferred@2064428585 after retry RPC after error@155510905) -> (continuation of Deferred@1029014978 after retry RPC after error@1033847851)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.8.27.252:39249,127.8.27.253:44545,127.8.27.254:34253) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.8.27.254:34253
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=26866), Traces: [0ms] refreshing cache from master, [33ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.252:39249, [115ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.253:44545, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.254:34253, [196ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.253:44545: Network error: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545, [246ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.254:34253: Network error: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [501ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.252:39249: OK, [506ms] delaying RPC due to: Service unavailable: Master config (127.8.27.252:39249,127.8.27.253:44545,127.8.27.254:34253) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [514ms] refreshing cache from master, [515ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.252:39249, [517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.253:44545, [519ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.254:34253, [520ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.253:44545: Network error: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545, [521ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.252:39249: OK, [524ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.254:34253: Network error: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [526ms] delaying RPC due to: Service unavailable: Master config (127.8.27.252:39249,127.8.27.253:44545,127.8.27.254:34253) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [535ms] refreshing cache from master, [535ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.252:39249, [535ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.253:44545, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.254:34253, [538ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.253:44545: Network error: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545, [540ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.252:39249: OK, [543ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.254:34253: Network error: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [544ms] delaying RPC due to: Service unavailable: Master config (127.8.27.252:39249,127.8.27.253:44545,127.8.27.254:34253) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [554ms] refreshing cache from master, [554ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.252:39249, [555ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.253:44545, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.254:34253, [559ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.253:44545: Network error: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545, [559ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.254:34253: Network error: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [561ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.252:39249: OK, [562ms] delaying RPC due to: Service unavailable: Master config (127.8.27.252:39249,127.8.27.253:44545,127.8.27.254:34253) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [574ms] refreshing cache from master, [574ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.252:39249, [574ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.253:44545, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.254:34253, [578ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.253:44545: Network error: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545, [578ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.254:34253: Network error: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [580ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.252:39249: OK, [581ms] delaying RPC due to: Service unavailable: Master config (127.8.27.252:39249,127.8.27.253:44545,127.8.27.254:34253) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545, [595ms] refreshing cache from master, [595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.252:39249, [595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.253:44545, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.254:34253, [600ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.253:44545: Network error: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545, [604ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.252:39249: OK, [605ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.254:34253: Network error: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [606ms] delaying RPC due to: Service unavailable: Master config (127.8.27.252:39249,127.8.27.253:44545,127.8.27.254:34253) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [655ms] refreshing cache from master, [655ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.252:39249, [655ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.253:44545, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.254:34253, [659ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.252:39249: OK, [659ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.253:44545: Network error: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545, [661ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.254:34253: Network error: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [663ms] delaying RPC due to: Service unavailable: Master config (127.8.27.252:39249,127.8.27.253:44545,127.8.27.254:34253) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [695ms] refreshing cache from master, [695ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.252:39249, [695ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.253:44545, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.254:34253, [698ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.253:44545: Network error: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545, [700ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.252:39249: OK, [702ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.254:34253: Network error: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [703ms] delaying RPC due to: Service unavailable: Master config (127.8.27.252:39249,127.8.27.253:44545,127.8.27.254:34253) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [714ms] refreshing cache from master, [714ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.252:39249, [715ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.253:44545, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.254:34253, [718ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.253:44545: Network error: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545, [719ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.252:39249: OK, [720ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.254:34253: Network error: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [721ms] delaying RPC due to: Service unavailable: Master config (127.8.27.252:39249,127.8.27.253:44545,127.8.27.254:34253) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [995ms] refreshing cache from master, [995ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.252:39249, [995ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.253:44545, [996ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.254:34253, [999ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.253:44545: Network error: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545, [999ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.252:39249: OK, [1000ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.254:34253: Network error: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [1001ms] delaying RPC due to: Service unavailable: Master config (127.8.27.252:39249,127.8.27.253:44545,127.8.27.254:34253) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [1634ms] refreshing cache from master, [1634ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.252:39249, [1635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.253:44545, [1635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.254:34253, [1638ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.253:44545: Network error: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545, [1639ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.252:39249: OK, [1639ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.254:34253: Network error: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [1639ms] delaying RPC due to: Service unavailable: Master config (127.8.27.252:39249,127.8.27.253:44545,127.8.27.254:34253) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [2994ms] refreshing cache from master, [2994ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.252:39249, [2995ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.253:44545, [2996ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.254:34253, [2999ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.253:44545: Network error: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545, [2999ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.252:39249: OK, [3000ms] Sub RPC ConnectToMaster: received response from server master-127.8.27.254:34253: Network error: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [3000ms] delaying RPC due to: Service unavailable: Master config (127.8.27.252:39249,127.8.27.253:44545,127.8.27.254:34253) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253, [3294ms] refreshing cache from master, [3294ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.252:39249, [3295ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.253:44545, [3296ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.27.254:34253, [3298ms] too many traces: truncated at 100 traces, deferred=Deferred@595595211(state=PENDING, result=null, callback=(continuation of Deferred@1254517119 after retry RPC after error@109429248) -> (continuation of Deferred@895107142 after retry RPC after error@378007409) -> (continuation of Deferred@1878257999 after retry RPC after error@1204730739) -> (continuation of Deferred@544316348 after retry RPC after error@161748132) -> (continuation of Deferred@2041443670 after retry RPC after error@1267391539) -> (continuation of Deferred@1728713349 after retry RPC after error@1319255028) -> (continuation of Deferred@1479766067 after retry RPC after error@1170555970) -> (continuation of Deferred@1928128581 after retry RPC after error@768479351) -> (continuation of Deferred@1849646869 after retry RPC after error@28832196) -> (continuation of Deferred@344756087 after retry RPC after error@1226598457) -> (continuation of Deferred@1573863928 after retry RPC after error@1674097236) -> (continuation of Deferred@323972717 after retry RPC after error@960004509) -> (continuation of Deferred@509597635 after retry RPC after error@950246478) -> (continuation of Deferred@1350021425 after retry RPC after error@759097035) -> (continuation of Deferred@1296026976 after retry RPC after error@1775925605) -> (continuation of Deferred@80327510 after retry RPC after error@1578833338) -> (continuation of Deferred@1836877780 after retry RPC after error@1473759823) -> (continuation of Deferred@508395744 after retry RPC after error@410906710) -> (continuation of Deferred@297174698 after retry RPC after error@2019210081) -> (continuation of Deferred@174224906 after retry RPC after error@2112538704) -> (continuation of Deferred@1489801873 after retry RPC after error@1973482322) -> (continuation of Deferred@1818954186 after retry RPC after error@1206445151) -> (continuation of Deferred@2064428585 after retry RPC after error@155510905) -> (continuation of Deferred@1029014978 after retry RPC after error@1033847851), errback=(continuation of Deferred@1254517119 after retry RPC after error@109429248) -> (continuation of Deferred@895107142 after retry RPC after error@378007409) -> (continuation of Deferred@1878257999 after retry RPC after error@1204730739) -> (continuation of Deferred@544316348 after retry RPC after error@161748132) -> (continuation of Deferred@2041443670 after retry RPC after error@1267391539) -> (continuation of Deferred@1728713349 after retry RPC after error@1319255028) -> (continuation of Deferred@1479766067 after retry RPC after error@1170555970) -> (continuation of Deferred@1928128581 after retry RPC after error@768479351) -> (continuation of Deferred@1849646869 after retry RPC after error@28832196) -> (continuation of Deferred@344756087 after retry RPC after error@1226598457) -> (continuation of Deferred@1573863928 after retry RPC after error@1674097236) -> (continuation of Deferred@323972717 after retry RPC after error@960004509) -> (continuation of Deferred@509597635 after retry RPC after error@950246478) -> (continuation of Deferred@1350021425 after retry RPC after error@759097035) -> (continuation of Deferred@1296026976 after retry RPC after error@1775925605) -> (continuation of Deferred@80327510 after retry RPC after error@1578833338) -> (continuation of Deferred@1836877780 after retry RPC after error@1473759823) -> (continuation of Deferred@508395744 after retry RPC after error@410906710) -> (continuation of Deferred@297174698 after retry RPC after error@2019210081) -> (continuation of Deferred@174224906 after retry RPC after error@2112538704) -> (continuation of Deferred@1489801873 after retry RPC after error@1973482322) -> (continuation of Deferred@1818954186 after retry RPC after error@1206445151) -> (continuation of Deferred@2064428585 after retry RPC after error@155510905) -> (continuation of Deferred@1029014978 after retry RPC after error@1033847851)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.8.27.252:39249,127.8.27.253:44545,127.8.27.254:34253) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.253:44545(127.8.27.253:44545): Connection refused: /127.8.27.253:44545,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.8.27.254:34253(127.8.27.254:34253): Connection refused: /127.8.27.254:34253
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.8.27.254:34253
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)