23:30:49.802 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
23:30:49.813 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskpPnkEK/test-tmp
23:30:49.815 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
23:30:49.818 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskpPnkEK/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
23:30:50.021 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
23:30:50.194 [DEBUG - main] (MiniKuduCluster.java:195) Response:
23:30:50.195 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
23:30:50.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:50.199631 1751 mini_oidc.cc:152] Starting JWKS server
23:30:50.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:50.203840 1751 webserver.cc:460] Webserver started at http://127.0.0.1:43935/ using document root <none> and password file <none>
23:30:50.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:50.204362 1751 mini_oidc.cc:189] Starting OIDC Discovery server
23:30:50.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:50.206100 1751 webserver.cc:460] Webserver started at http://127.0.0.1:36387/ using document root <none> and password file <none>
23:30:50.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T23:30:50Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
23:30:50.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T23:30:50Z Disabled control of system clock
23:30:50.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:50.252009 1751 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskpPnkEK/build/tsan/bin/kudu
23:30:50.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskpPnkEK/build/tsan/bin/kudu
23:30:50.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/master-0/wal
23:30:50.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/master-0/data
23:30:50.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:30:50.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:30:50.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:30:50.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:30:50.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:30:50.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:30:50.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/master-0/logs
23:30:50.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/master-0/data/info.pb
23:30:50.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:30:50.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:30:50.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:30:50.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:30:50.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:30:50.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:30:50.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:30:50.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:30:50.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
23:30:50.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:30:50.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:30:50.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:30:50.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.181.254:36363
23:30:50.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.181.254
23:30:50.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:30:50.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.181.212:42595
23:30:50.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:30:50.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:30:50.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:30:50.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:30:50.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.181.254:36363,127.1.181.253:46667,127.1.181.252:44487
23:30:50.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:30:50.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43935/jwks.json with env {}
23:30:50.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:50.547060 1763 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:30:50.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:50.547595 1763 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:30:50.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:50.548314 1763 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:30:50.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:50.578123 1763 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
23:30:50.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:50.578377 1763 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:30:50.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:50.578559 1763 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
23:30:50.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:50.578796 1763 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
23:30:50.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:50.579089 1763 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
23:30:50.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:50.579305 1763 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43935/jwks.json
23:30:50.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:50.613075 1763 master_runner.cc:386] Master server non-default flags:
23:30:50.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:30:50.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.181.212:42595
23:30:50.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:30:50.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:30:50.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/master-0/data
23:30:50.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/master-0/wal
23:30:50.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:30:50.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.181.254:36363,127.1.181.253:46667,127.1.181.252:44487
23:30:50.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:30:50.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:30:50.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:30:50.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.181.254:36363
23:30:50.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:30:50.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:30:50.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:30:50.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43935/jwks.json
23:30:50.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:30:50.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:30:50.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/master-0/data/info.pb
23:30:50.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.181.254
23:30:50.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:30:50.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:30:50.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:30:50.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:30:50.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:30:50.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:30:50.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/master-0/logs
23:30:50.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:30:50.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:30:50.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
23:30:50.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
23:30:50.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:30:50.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:30:50.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:30:50.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 22:42:10 UTC on ccycloud.quasar-yawxpd.root.comops.site
23:30:50.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676391
23:30:50.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:30:50.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:50.614462 1763 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:30:50.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:50.616605 1763 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:30:50.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:50.636145 1770 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:30:51.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:50.638361 1763 server_base.cc:1034] running on GCE node
23:30:51.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:50.636219 1769 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:30:51.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:50.636458 1772 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:30:51.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:51.823632 1763 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:30:51.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:51.826617 1763 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:30:51.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:51.827999 1763 hybrid_clock.cc:648] HybridClock initialized: now 1790551851827937 us; error 66 us; skew 500 ppm
23:30:51.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:51.829152 1763 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:30:51.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:51.836519 1763 webserver.cc:460] Webserver started at http://127.1.181.254:33993/ using document root <none> and password file <none>
23:30:51.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:51.837841 1763 fs_manager.cc:359] Metadata directory not provided
23:30:51.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:51.838335 1763 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:30:51.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:51.839013 1763 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
23:30:51.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:51.846352 1763 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/master-0/data/instance:
23:30:51.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8624cf9c568e48ffa151e70ee3acd89a"
23:30:51.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 23:30:51 on dist-test-slave-dnlc"
23:30:51.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:51.847848 1763 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/master-0/wal/instance:
23:30:51.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8624cf9c568e48ffa151e70ee3acd89a"
23:30:51.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 23:30:51 on dist-test-slave-dnlc"
23:30:51.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:51.857741 1763 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.011s sys 0.000s
23:30:51.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:51.863149 1779 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
23:30:51.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:51.864182 1763 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.005s sys 0.000s
23:30:51.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:51.864528 1763 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/master-0/data,/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/master-0/wal
23:30:51.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8624cf9c568e48ffa151e70ee3acd89a"
23:30:51.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 23:30:51 on dist-test-slave-dnlc"
23:30:51.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:51.864916 1763 fs_report.cc:383] FS layout report
23:30:51.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:30:51.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/master-0/wal
23:30:51.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/master-0/wal
23:30:51.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/master-0/data/data
23:30:51.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:30:51.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:30:51.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:30:51.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:30:51.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:30:51.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:30:51.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:30:51.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:30:51.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:30:51.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:30:51.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:30:51.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:51.933586 1763 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
23:30:51.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:51.935223 1763 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:30:51.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:51.935745 1763 kserver.cc:163] Server-wide thread pool size limit: 3276
23:30:52.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:52.007812 1763 rpc_server.cc:307] RPC server started. Bound to: 127.1.181.254:36363
23:30:52.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:52.007895 1830 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.181.254:36363 every 8 connection(s)
23:30:52.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:52.010602 1763 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/master-0/data/info.pb
23:30:52.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:52.015764 1831 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
23:30:52.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:52.019691 1751 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskpPnkEK/build/tsan/bin/kudu as pid 1763
23:30:52.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:52.020192 1751 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/master-0/wal/instance
23:30:52.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:52.025262 1751 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskpPnkEK/build/tsan/bin/kudu
23:30:52.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskpPnkEK/build/tsan/bin/kudu
23:30:52.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/master-1/wal
23:30:52.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/master-1/data
23:30:52.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:30:52.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:30:52.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:30:52.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:30:52.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:30:52.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:30:52.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/master-1/logs
23:30:52.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/master-1/data/info.pb
23:30:52.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:30:52.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:30:52.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:30:52.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:30:52.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:30:52.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:30:52.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:30:52.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:30:52.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
23:30:52.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:30:52.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:30:52.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:30:52.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.181.253:46667
23:30:52.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.181.253
23:30:52.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:30:52.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.181.212:42595
23:30:52.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:30:52.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:30:52.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:30:52.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:30:52.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.181.254:36363,127.1.181.253:46667,127.1.181.252:44487
23:30:52.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:30:52.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43935/jwks.json with env {}
23:30:52.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:52.031900 1831 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.181.254" port: 36363 } has no permanent_uuid. Determining permanent_uuid...
23:30:52.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:52.065348 1831 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.181.253" port: 46667 } has no permanent_uuid. Determining permanent_uuid...
23:30:52.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:52.067458 1780 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.181.253:46667: connect: Connection refused (error 111)
23:30:52.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:52.071784 1831 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.181.253:46667: Network error: Client connection negotiation failed: client connection to 127.1.181.253:46667: connect: Connection refused (error 111)
23:30:52.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:52.121275 1831 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.181.253" port: 46667 } attempt: 1
23:30:52.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:52.125490 1831 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.181.253:46667: Network error: Client connection negotiation failed: client connection to 127.1.181.253:46667: connect: Connection refused (error 111)
23:30:52.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:52.193975 1831 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.181.253" port: 46667 } attempt: 2
23:30:52.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:52.198201 1831 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.181.253:46667: Network error: Client connection negotiation failed: client connection to 127.1.181.253:46667: connect: Connection refused (error 111)
23:30:52.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:52.289638 1831 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.181.253" port: 46667 } attempt: 3
23:30:52.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:52.293718 1831 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.181.253:46667: Network error: Client connection negotiation failed: client connection to 127.1.181.253:46667: connect: Connection refused (error 111)
23:30:52.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:52.337018 1833 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:30:52.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:52.337535 1833 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:30:52.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:52.338295 1833 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:30:52.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:52.368251 1833 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
23:30:52.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:52.368501 1833 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:30:52.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:52.368678 1833 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
23:30:52.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:52.368876 1833 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
23:30:52.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:52.369076 1833 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
23:30:52.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:52.369266 1833 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43935/jwks.json
23:30:52.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:52.403910 1833 master_runner.cc:386] Master server non-default flags:
23:30:52.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:30:52.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.181.212:42595
23:30:52.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:30:52.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:30:52.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/master-1/data
23:30:52.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/master-1/wal
23:30:52.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:30:52.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.181.254:36363,127.1.181.253:46667,127.1.181.252:44487
23:30:52.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:30:52.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:30:52.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:30:52.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.181.253:46667
23:30:52.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:30:52.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:30:52.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:30:52.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43935/jwks.json
23:30:52.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:30:52.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:30:52.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/master-1/data/info.pb
23:30:52.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.181.253
23:30:52.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:30:52.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:30:52.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:30:52.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:30:52.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:30:52.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:30:52.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/master-1/logs
23:30:52.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:30:52.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:30:52.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
23:30:52.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
23:30:52.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:30:52.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:30:52.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:30:52.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 22:42:10 UTC on ccycloud.quasar-yawxpd.root.comops.site
23:30:52.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676391
23:30:52.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:30:52.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:52.405020 1833 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:30:52.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:52.406800 1833 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:30:52.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:52.425189 1843 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:30:52.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:52.437109 1831 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.181.253" port: 46667 } attempt: 4
23:30:52.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:52.441524 1831 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.181.253:46667: Network error: Client connection negotiation failed: client connection to 127.1.181.253:46667: connect: Connection refused (error 111)
23:30:52.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:52.741024 1831 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.181.253" port: 46667 } attempt: 5
23:30:52.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:52.747144 1831 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.181.253:46667: Network error: Client connection negotiation failed: client connection to 127.1.181.253:46667: connect: Connection refused (error 111)
23:30:53.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:53.294654 1831 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.181.253" port: 46667 } attempt: 6
23:30:53.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:53.305351 1831 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.181.253:46667: Network error: Client connection negotiation failed: client connection to 127.1.181.253:46667: connect: Connection refused (error 111)
23:30:53.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:52.425311 1842 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:30:53.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:52.425309 1845 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:30:53.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:53.662596 1844 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1235 milliseconds
23:30:53.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:53.662716 1833 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
23:30:53.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:53.664134 1833 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:30:53.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:53.666858 1833 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:30:53.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:53.668246 1833 hybrid_clock.cc:648] HybridClock initialized: now 1790551853668216 us; error 39 us; skew 500 ppm
23:30:53.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:53.669211 1833 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:30:53.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:53.675776 1833 webserver.cc:460] Webserver started at http://127.1.181.253:41099/ using document root <none> and password file <none>
23:30:53.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:53.676781 1833 fs_manager.cc:359] Metadata directory not provided
23:30:53.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:53.677026 1833 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:30:53.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:53.677600 1833 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
23:30:53.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:53.682807 1833 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/master-1/data/instance:
23:30:53.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "726ed7c14b38409ca79aef03c750f165"
23:30:53.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 23:30:53 on dist-test-slave-dnlc"
23:30:53.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:53.684027 1833 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/master-1/wal/instance:
23:30:53.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "726ed7c14b38409ca79aef03c750f165"
23:30:53.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 23:30:53 on dist-test-slave-dnlc"
23:30:53.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:53.691254 1833 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.005s sys 0.003s
23:30:53.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:53.697146 1854 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
23:30:53.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:53.698159 1833 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
23:30:53.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:53.698567 1833 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/master-1/data,/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/master-1/wal
23:30:53.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "726ed7c14b38409ca79aef03c750f165"
23:30:53.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 23:30:53 on dist-test-slave-dnlc"
23:30:53.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:53.698915 1833 fs_report.cc:383] FS layout report
23:30:53.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:30:53.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/master-1/wal
23:30:53.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/master-1/wal
23:30:53.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/master-1/data/data
23:30:53.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:30:53.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:30:53.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:30:53.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:30:53.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:30:53.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:30:53.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:30:53.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:30:53.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:30:53.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:30:53.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:30:53.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:53.754118 1833 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
23:30:53.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:53.755613 1833 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:30:53.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:53.756065 1833 kserver.cc:163] Server-wide thread pool size limit: 3276
23:30:53.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:53.829057 1833 rpc_server.cc:307] RPC server started. Bound to: 127.1.181.253:46667
23:30:53.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:53.829144 1905 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.181.253:46667 every 8 connection(s)
23:30:53.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:53.831667 1833 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/master-1/data/info.pb
23:30:53.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:53.836714 1906 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
23:30:53.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:53.838267 1751 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskpPnkEK/build/tsan/bin/kudu as pid 1833
23:30:53.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:53.838680 1751 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/master-1/wal/instance
23:30:53.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:53.840627 1751 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskpPnkEK/build/tsan/bin/kudu
23:30:53.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskpPnkEK/build/tsan/bin/kudu
23:30:53.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/master-2/wal
23:30:53.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/master-2/data
23:30:53.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:30:53.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:30:53.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:30:53.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:30:53.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:30:53.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:30:53.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/master-2/logs
23:30:53.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/master-2/data/info.pb
23:30:53.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:30:53.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:30:53.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:30:53.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:30:53.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:30:53.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:30:53.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:30:53.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:30:53.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
23:30:53.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:30:53.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:30:53.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:30:53.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.181.252:44487
23:30:53.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.181.252
23:30:53.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:30:53.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.181.212:42595
23:30:53.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:30:53.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:30:53.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:30:53.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:30:53.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.181.254:36363,127.1.181.253:46667,127.1.181.252:44487
23:30:53.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:30:53.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43935/jwks.json with env {}
23:30:53.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:53.850701 1906 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.181.254" port: 36363 } has no permanent_uuid. Determining permanent_uuid...
23:30:53.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:53.876446 1906 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.181.253" port: 46667 } has no permanent_uuid. Determining permanent_uuid...
23:30:53.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:53.892729 1906 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.181.252" port: 44487 } has no permanent_uuid. Determining permanent_uuid...
23:30:53.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:53.895998 1856 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.181.252:44487: connect: Connection refused (error 111)
23:30:53.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:53.900434 1906 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.181.252:44487: Network error: Client connection negotiation failed: client connection to 127.1.181.252:44487: connect: Connection refused (error 111)
23:30:53.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:53.949862 1906 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.181.252" port: 44487 } attempt: 1
23:30:53.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:53.953980 1906 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.181.252:44487: Network error: Client connection negotiation failed: client connection to 127.1.181.252:44487: connect: Connection refused (error 111)
23:30:54.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:54.022343 1906 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.181.252" port: 44487 } attempt: 2
23:30:54.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:54.026966 1906 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.181.252:44487: Network error: Client connection negotiation failed: client connection to 127.1.181.252:44487: connect: Connection refused (error 111)
23:30:54.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:54.118403 1906 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.181.252" port: 44487 } attempt: 3
23:30:54.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:54.121958 1906 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.181.252:44487: Network error: Client connection negotiation failed: client connection to 127.1.181.252:44487: connect: Connection refused (error 111)
23:30:54.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:54.160955 1908 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:30:54.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:54.161448 1908 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:30:54.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:54.161969 1908 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:30:54.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:54.192353 1908 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
23:30:54.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:54.192618 1908 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:30:54.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:54.192801 1908 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
23:30:54.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:54.193090 1908 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
23:30:54.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:54.193363 1908 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
23:30:54.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:54.193532 1908 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43935/jwks.json
23:30:54.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:54.228857 1908 master_runner.cc:386] Master server non-default flags:
23:30:54.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:30:54.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.181.212:42595
23:30:54.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:30:54.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:30:54.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/master-2/data
23:30:54.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/master-2/wal
23:30:54.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:30:54.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.181.254:36363,127.1.181.253:46667,127.1.181.252:44487
23:30:54.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:30:54.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:30:54.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:30:54.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.181.252:44487
23:30:54.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:30:54.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:30:54.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:30:54.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43935/jwks.json
23:30:54.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:30:54.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:30:54.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/master-2/data/info.pb
23:30:54.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.181.252
23:30:54.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:30:54.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:30:54.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:30:54.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:30:54.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:30:54.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:30:54.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/master-2/logs
23:30:54.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:30:54.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:30:54.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
23:30:54.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
23:30:54.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:30:54.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:30:54.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:30:54.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 22:42:10 UTC on ccycloud.quasar-yawxpd.root.comops.site
23:30:54.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676391
23:30:54.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:30:54.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:54.230266 1908 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:30:54.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:54.232417 1908 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:30:54.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:54.254632 1919 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:30:54.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:54.265393 1906 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.181.252" port: 44487 } attempt: 4
23:30:54.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:54.270218 1906 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.181.252:44487: Network error: Client connection negotiation failed: client connection to 127.1.181.252:44487: connect: Connection refused (error 111)
23:30:54.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:54.365890 1831 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.181.253" port: 46667 } attempt: 7
23:30:54.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:54.381834 1831 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.181.252" port: 44487 } has no permanent_uuid. Determining permanent_uuid...
23:30:54.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:54.388266 1831 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.181.252:44487: Network error: Client connection negotiation failed: client connection to 127.1.181.252:44487: connect: Connection refused (error 111)
23:30:54.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:54.446866 1831 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.181.252" port: 44487 } attempt: 1
23:30:54.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:54.450723 1831 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.181.252:44487: Network error: Client connection negotiation failed: client connection to 127.1.181.252:44487: connect: Connection refused (error 111)
23:30:54.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:54.532335 1831 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.181.252" port: 44487 } attempt: 2
23:30:54.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:54.536868 1831 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.181.252:44487: Network error: Client connection negotiation failed: client connection to 127.1.181.252:44487: connect: Connection refused (error 111)
23:30:54.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:54.569720 1906 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.181.252" port: 44487 } attempt: 5
23:30:54.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:54.573113 1906 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.181.252:44487: Network error: Client connection negotiation failed: client connection to 127.1.181.252:44487: connect: Connection refused (error 111)
23:30:54.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:54.622448 1831 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.181.252" port: 44487 } attempt: 3
23:30:54.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:54.626511 1831 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.181.252:44487: Network error: Client connection negotiation failed: client connection to 127.1.181.252:44487: connect: Connection refused (error 111)
23:30:54.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:54.767089 1831 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.181.252" port: 44487 } attempt: 4
23:30:54.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:54.771382 1831 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.181.252:44487: Network error: Client connection negotiation failed: client connection to 127.1.181.252:44487: connect: Connection refused (error 111)
23:30:55.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:55.054934 1831 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.181.252" port: 44487 } attempt: 5
23:30:55.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:55.059113 1831 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.181.252:44487: Network error: Client connection negotiation failed: client connection to 127.1.181.252:44487: connect: Connection refused (error 111)
23:30:55.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:55.120498 1906 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.181.252" port: 44487 } attempt: 6
23:30:55.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:55.127321 1906 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.181.252:44487: Network error: Client connection negotiation failed: client connection to 127.1.181.252:44487: connect: Connection refused (error 111)
23:30:55.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:54.255815 1921 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:30:55.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:54.254683 1918 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:30:55.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:54.256868 1908 server_base.cc:1034] running on GCE node
23:30:55.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:55.462266 1908 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:30:55.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:55.464668 1908 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:30:55.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:55.465996 1908 hybrid_clock.cc:648] HybridClock initialized: now 1790551855465964 us; error 49 us; skew 500 ppm
23:30:55.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:55.466830 1908 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:30:55.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:55.473186 1908 webserver.cc:460] Webserver started at http://127.1.181.252:40067/ using document root <none> and password file <none>
23:30:55.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:55.474107 1908 fs_manager.cc:359] Metadata directory not provided
23:30:55.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:55.474282 1908 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:30:55.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:55.474704 1908 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
23:30:55.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:55.481602 1908 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/master-2/data/instance:
23:30:55.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "42441f8110d24b78a8ea9b73a71776d9"
23:30:55.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 23:30:55 on dist-test-slave-dnlc"
23:30:55.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:55.483048 1908 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/master-2/wal/instance:
23:30:55.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "42441f8110d24b78a8ea9b73a71776d9"
23:30:55.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 23:30:55 on dist-test-slave-dnlc"
23:30:55.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:55.491863 1908 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.006s sys 0.003s
23:30:55.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:55.497411 1932 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
23:30:55.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:55.498593 1908 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.001s sys 0.002s
23:30:55.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:55.498875 1908 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/master-2/data,/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/master-2/wal
23:30:55.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "42441f8110d24b78a8ea9b73a71776d9"
23:30:55.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 23:30:55 on dist-test-slave-dnlc"
23:30:55.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:55.499162 1908 fs_report.cc:383] FS layout report
23:30:55.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:30:55.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/master-2/wal
23:30:55.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/master-2/wal
23:30:55.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/master-2/data/data
23:30:55.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:30:55.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:30:55.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:30:55.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:30:55.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:30:55.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:30:55.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:30:55.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:30:55.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:30:55.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:30:55.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:30:55.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:55.543100 1908 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
23:30:55.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:55.544445 1908 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:30:55.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:55.544824 1908 kserver.cc:163] Server-wide thread pool size limit: 3276
23:30:55.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:55.611721 1831 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.181.252" port: 44487 } attempt: 6
23:30:55.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:55.619297 1831 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.181.252:44487: Network error: Client connection negotiation failed: client connection to 127.1.181.252:44487: connect: Connection refused (error 111)
23:30:55.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:55.620885 1908 rpc_server.cc:307] RPC server started. Bound to: 127.1.181.252:44487
23:30:55.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:55.621129 1985 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.181.252:44487 every 8 connection(s)
23:30:55.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:55.623800 1908 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/master-2/data/info.pb
23:30:55.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:55.629612 1751 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskpPnkEK/build/tsan/bin/kudu as pid 1908
23:30:55.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:55.630102 1751 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/master-2/wal/instance
23:30:55.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:55.630487 1986 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
23:30:55.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:55.649291 1986 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.181.254" port: 36363 } has no permanent_uuid. Determining permanent_uuid...
23:30:55.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:55.676180 1986 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.181.253" port: 46667 } has no permanent_uuid. Determining permanent_uuid...
23:30:55.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:55.687295 1986 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.181.252" port: 44487 } has no permanent_uuid. Determining permanent_uuid...
23:30:55.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:55.704018 1986 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 42441f8110d24b78a8ea9b73a71776d9: Bootstrap starting.
23:30:55.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:55.710455 1986 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 42441f8110d24b78a8ea9b73a71776d9: Neither blocks nor log segments found. Creating new log.
23:30:55.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:55.712085 1986 log.cc:824] T 00000000000000000000000000000000 P 42441f8110d24b78a8ea9b73a71776d9: Log is configured to *not* fsync() on all Append() calls
23:30:55.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:55.716895 1986 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 42441f8110d24b78a8ea9b73a71776d9: No bootstrap required, opened a new log
23:30:55.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:55.738736 1986 raft_consensus.cc:348] T 00000000000000000000000000000000 P 42441f8110d24b78a8ea9b73a71776d9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8624cf9c568e48ffa151e70ee3acd89a" member_type: VOTER last_known_addr { host: "127.1.181.254" port: 36363 } } peers { permanent_uuid: "726ed7c14b38409ca79aef03c750f165" member_type: VOTER last_known_addr { host: "127.1.181.253" port: 46667 } } peers { permanent_uuid: "42441f8110d24b78a8ea9b73a71776d9" member_type: VOTER last_known_addr { host: "127.1.181.252" port: 44487 } }
23:30:55.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:55.739321 1986 raft_consensus.cc:374] T 00000000000000000000000000000000 P 42441f8110d24b78a8ea9b73a71776d9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
23:30:55.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:55.739504 1986 raft_consensus.cc:729] T 00000000000000000000000000000000 P 42441f8110d24b78a8ea9b73a71776d9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 42441f8110d24b78a8ea9b73a71776d9, State: Initialized, Role: FOLLOWER
23:30:55.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:55.740113 1986 consensus_queue.cc:260] T 00000000000000000000000000000000 P 42441f8110d24b78a8ea9b73a71776d9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8624cf9c568e48ffa151e70ee3acd89a" member_type: VOTER last_known_addr { host: "127.1.181.254" port: 36363 } } peers { permanent_uuid: "726ed7c14b38409ca79aef03c750f165" member_type: VOTER last_known_addr { host: "127.1.181.253" port: 46667 } } peers { permanent_uuid: "42441f8110d24b78a8ea9b73a71776d9" member_type: VOTER last_known_addr { host: "127.1.181.252" port: 44487 } }
23:30:55.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:55.742617 1995 sys_catalog.cc:455] T 00000000000000000000000000000000 P 42441f8110d24b78a8ea9b73a71776d9 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8624cf9c568e48ffa151e70ee3acd89a" member_type: VOTER last_known_addr { host: "127.1.181.254" port: 36363 } } peers { permanent_uuid: "726ed7c14b38409ca79aef03c750f165" member_type: VOTER last_known_addr { host: "127.1.181.253" port: 46667 } } peers { permanent_uuid: "42441f8110d24b78a8ea9b73a71776d9" member_type: VOTER last_known_addr { host: "127.1.181.252" port: 44487 } } }
23:30:55.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:55.743291 1995 sys_catalog.cc:458] T 00000000000000000000000000000000 P 42441f8110d24b78a8ea9b73a71776d9 [sys.catalog]: This master's current role is: FOLLOWER
23:30:55.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:55.743970 1986 sys_catalog.cc:564] T 00000000000000000000000000000000 P 42441f8110d24b78a8ea9b73a71776d9 [sys.catalog]: configured and running, proceeding with master startup.
23:30:55.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:55.770609 2006 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 42441f8110d24b78a8ea9b73a71776d9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:30:55.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:55.770924 2006 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:30:55.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:55.766489 1908 master_runner.cc:186] Error getting master registration for 127.1.181.254:36363: OK, instance_id { permanent_uuid: "8624cf9c568e48ffa151e70ee3acd89a" instance_seqno: 1790551851987903 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
23:30:55.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:55.790629 1908 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
23:30:55.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:55.800191 1938 tablet.cc:2369] T 00000000000000000000000000000000 P 42441f8110d24b78a8ea9b73a71776d9: Can't schedule compaction. Clean time has not been advanced past its initial value.
23:30:56.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:56.151535 1995 raft_consensus.cc:482] T 00000000000000000000000000000000 P 42441f8110d24b78a8ea9b73a71776d9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
23:30:56.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:56.152107 1995 raft_consensus.cc:504] T 00000000000000000000000000000000 P 42441f8110d24b78a8ea9b73a71776d9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8624cf9c568e48ffa151e70ee3acd89a" member_type: VOTER last_known_addr { host: "127.1.181.254" port: 36363 } } peers { permanent_uuid: "726ed7c14b38409ca79aef03c750f165" member_type: VOTER last_known_addr { host: "127.1.181.253" port: 46667 } } peers { permanent_uuid: "42441f8110d24b78a8ea9b73a71776d9" member_type: VOTER last_known_addr { host: "127.1.181.252" port: 44487 } }
23:30:56.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:56.154542 1995 leader_election.cc:288] T 00000000000000000000000000000000 P 42441f8110d24b78a8ea9b73a71776d9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8624cf9c568e48ffa151e70ee3acd89a (127.1.181.254:36363), 726ed7c14b38409ca79aef03c750f165 (127.1.181.253:46667)
23:30:56.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:56.187860 1906 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.181.252" port: 44487 } attempt: 7
23:30:56.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:56.210522 1906 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 726ed7c14b38409ca79aef03c750f165: Bootstrap starting.
23:30:56.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:56.218086 1906 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 726ed7c14b38409ca79aef03c750f165: Neither blocks nor log segments found. Creating new log.
23:30:56.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:56.219873 1906 log.cc:824] T 00000000000000000000000000000000 P 726ed7c14b38409ca79aef03c750f165: Log is configured to *not* fsync() on all Append() calls
23:30:56.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:56.225452 1906 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 726ed7c14b38409ca79aef03c750f165: No bootstrap required, opened a new log
23:30:56.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:56.253334 1906 raft_consensus.cc:348] T 00000000000000000000000000000000 P 726ed7c14b38409ca79aef03c750f165 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8624cf9c568e48ffa151e70ee3acd89a" member_type: VOTER last_known_addr { host: "127.1.181.254" port: 36363 } } peers { permanent_uuid: "726ed7c14b38409ca79aef03c750f165" member_type: VOTER last_known_addr { host: "127.1.181.253" port: 46667 } } peers { permanent_uuid: "42441f8110d24b78a8ea9b73a71776d9" member_type: VOTER last_known_addr { host: "127.1.181.252" port: 44487 } }
23:30:56.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:56.254148 1906 raft_consensus.cc:374] T 00000000000000000000000000000000 P 726ed7c14b38409ca79aef03c750f165 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
23:30:56.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:56.254386 1906 raft_consensus.cc:729] T 00000000000000000000000000000000 P 726ed7c14b38409ca79aef03c750f165 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 726ed7c14b38409ca79aef03c750f165, State: Initialized, Role: FOLLOWER
23:30:56.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:56.255348 1906 consensus_queue.cc:260] T 00000000000000000000000000000000 P 726ed7c14b38409ca79aef03c750f165 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8624cf9c568e48ffa151e70ee3acd89a" member_type: VOTER last_known_addr { host: "127.1.181.254" port: 36363 } } peers { permanent_uuid: "726ed7c14b38409ca79aef03c750f165" member_type: VOTER last_known_addr { host: "127.1.181.253" port: 46667 } } peers { permanent_uuid: "42441f8110d24b78a8ea9b73a71776d9" member_type: VOTER last_known_addr { host: "127.1.181.252" port: 44487 } }
23:30:56.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:56.260926 2010 sys_catalog.cc:455] T 00000000000000000000000000000000 P 726ed7c14b38409ca79aef03c750f165 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8624cf9c568e48ffa151e70ee3acd89a" member_type: VOTER last_known_addr { host: "127.1.181.254" port: 36363 } } peers { permanent_uuid: "726ed7c14b38409ca79aef03c750f165" member_type: VOTER last_known_addr { host: "127.1.181.253" port: 46667 } } peers { permanent_uuid: "42441f8110d24b78a8ea9b73a71776d9" member_type: VOTER last_known_addr { host: "127.1.181.252" port: 44487 } } }
23:30:56.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:56.261701 2010 sys_catalog.cc:458] T 00000000000000000000000000000000 P 726ed7c14b38409ca79aef03c750f165 [sys.catalog]: This master's current role is: FOLLOWER
23:30:56.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:56.262426 1906 sys_catalog.cc:564] T 00000000000000000000000000000000 P 726ed7c14b38409ca79aef03c750f165 [sys.catalog]: configured and running, proceeding with master startup.
23:30:56.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:56.262600 1881 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "42441f8110d24b78a8ea9b73a71776d9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "726ed7c14b38409ca79aef03c750f165" is_pre_election: true
23:30:56.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:56.263090 1860 tablet.cc:2369] T 00000000000000000000000000000000 P 726ed7c14b38409ca79aef03c750f165: Can't schedule compaction. Clean time has not been advanced past its initial value.
23:30:56.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:56.264021 1881 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 726ed7c14b38409ca79aef03c750f165 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 42441f8110d24b78a8ea9b73a71776d9 in term 0.
23:30:56.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:56.265103 1933 leader_election.cc:302] T 00000000000000000000000000000000 P 42441f8110d24b78a8ea9b73a71776d9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 42441f8110d24b78a8ea9b73a71776d9, 726ed7c14b38409ca79aef03c750f165; no voters:
23:30:56.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:56.266132 1995 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 42441f8110d24b78a8ea9b73a71776d9 [term 0 FOLLOWER]: Leader pre-election won for term 1
23:30:56.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:56.266449 1995 raft_consensus.cc:482] T 00000000000000000000000000000000 P 42441f8110d24b78a8ea9b73a71776d9 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
23:30:56.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:56.266749 1995 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 42441f8110d24b78a8ea9b73a71776d9 [term 0 FOLLOWER]: Advancing to term 1
23:30:56.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:56.272706 1995 raft_consensus.cc:504] T 00000000000000000000000000000000 P 42441f8110d24b78a8ea9b73a71776d9 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8624cf9c568e48ffa151e70ee3acd89a" member_type: VOTER last_known_addr { host: "127.1.181.254" port: 36363 } } peers { permanent_uuid: "726ed7c14b38409ca79aef03c750f165" member_type: VOTER last_known_addr { host: "127.1.181.253" port: 46667 } } peers { permanent_uuid: "42441f8110d24b78a8ea9b73a71776d9" member_type: VOTER last_known_addr { host: "127.1.181.252" port: 44487 } }
23:30:56.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:56.275558 1995 leader_election.cc:288] T 00000000000000000000000000000000 P 42441f8110d24b78a8ea9b73a71776d9 [CANDIDATE]: Term 1 election: Requested vote from peers 8624cf9c568e48ffa151e70ee3acd89a (127.1.181.254:36363), 726ed7c14b38409ca79aef03c750f165 (127.1.181.253:46667)
23:30:56.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:56.276046 1881 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "42441f8110d24b78a8ea9b73a71776d9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "726ed7c14b38409ca79aef03c750f165"
23:30:56.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:56.276657 1881 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 726ed7c14b38409ca79aef03c750f165 [term 0 FOLLOWER]: Advancing to term 1
23:30:56.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:56.284309 1881 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 726ed7c14b38409ca79aef03c750f165 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 42441f8110d24b78a8ea9b73a71776d9 in term 1.
23:30:56.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:56.285728 1933 leader_election.cc:302] T 00000000000000000000000000000000 P 42441f8110d24b78a8ea9b73a71776d9 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 42441f8110d24b78a8ea9b73a71776d9, 726ed7c14b38409ca79aef03c750f165; no voters:
23:30:56.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:56.286455 1995 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 42441f8110d24b78a8ea9b73a71776d9 [term 1 FOLLOWER]: Leader election won for term 1
23:30:56.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:56.307747 1995 raft_consensus.cc:686] T 00000000000000000000000000000000 P 42441f8110d24b78a8ea9b73a71776d9 [term 1 LEADER]: Becoming Leader. State: Replica: 42441f8110d24b78a8ea9b73a71776d9, State: Running, Role: LEADER
23:30:56.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:56.309105 1995 consensus_queue.cc:237] T 00000000000000000000000000000000 P 42441f8110d24b78a8ea9b73a71776d9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8624cf9c568e48ffa151e70ee3acd89a" member_type: VOTER last_known_addr { host: "127.1.181.254" port: 36363 } } peers { permanent_uuid: "726ed7c14b38409ca79aef03c750f165" member_type: VOTER last_known_addr { host: "127.1.181.253" port: 46667 } } peers { permanent_uuid: "42441f8110d24b78a8ea9b73a71776d9" member_type: VOTER last_known_addr { host: "127.1.181.252" port: 44487 } }
23:30:56.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:56.322263 2015 sys_catalog.cc:455] T 00000000000000000000000000000000 P 42441f8110d24b78a8ea9b73a71776d9 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 42441f8110d24b78a8ea9b73a71776d9. Latest consensus state: current_term: 1 leader_uuid: "42441f8110d24b78a8ea9b73a71776d9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8624cf9c568e48ffa151e70ee3acd89a" member_type: VOTER last_known_addr { host: "127.1.181.254" port: 36363 } } peers { permanent_uuid: "726ed7c14b38409ca79aef03c750f165" member_type: VOTER last_known_addr { host: "127.1.181.253" port: 46667 } } peers { permanent_uuid: "42441f8110d24b78a8ea9b73a71776d9" member_type: VOTER last_known_addr { host: "127.1.181.252" port: 44487 } } }
23:30:56.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:56.326121 2015 sys_catalog.cc:458] T 00000000000000000000000000000000 P 42441f8110d24b78a8ea9b73a71776d9 [sys.catalog]: This master's current role is: LEADER
23:30:56.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:56.329071 2024 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
23:30:56.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:56.336966 2024 catalog_manager.cc:1484] Initializing Kudu cluster ID...
23:30:56.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:56.345149 2023 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 726ed7c14b38409ca79aef03c750f165: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:30:56.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:56.345691 2023 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:30:56.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:56.368234 1881 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 726ed7c14b38409ca79aef03c750f165 [term 1 FOLLOWER]: Refusing update from remote peer 42441f8110d24b78a8ea9b73a71776d9: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
23:30:56.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:56.369663 2015 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 42441f8110d24b78a8ea9b73a71776d9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "726ed7c14b38409ca79aef03c750f165" member_type: VOTER last_known_addr { host: "127.1.181.253" port: 46667 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
23:30:56.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:56.329506 1833 master_runner.cc:186] Error getting master registration for 127.1.181.254:36363: OK, instance_id { permanent_uuid: "8624cf9c568e48ffa151e70ee3acd89a" instance_seqno: 1790551851987903 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
23:30:56.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:56.395462 1833 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
23:30:56.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:56.427882 2026 mvcc.cc:205] Tried to move back new op lower bound from 7334100403669217280 to 7334100403452960768. Current Snapshot: MvccSnapshot[applied={T|T < 7334100403669217280}]
23:30:56.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:56.432111 2024 catalog_manager.cc:1347] Generated new cluster ID: b175d83e22ab431c8acddd74bc0c42a2
23:30:56.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:56.432385 2024 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
23:30:56.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:56.432502 1995 sys_catalog.cc:455] T 00000000000000000000000000000000 P 42441f8110d24b78a8ea9b73a71776d9 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "42441f8110d24b78a8ea9b73a71776d9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8624cf9c568e48ffa151e70ee3acd89a" member_type: VOTER last_known_addr { host: "127.1.181.254" port: 36363 } } peers { permanent_uuid: "726ed7c14b38409ca79aef03c750f165" member_type: VOTER last_known_addr { host: "127.1.181.253" port: 46667 } } peers { permanent_uuid: "42441f8110d24b78a8ea9b73a71776d9" member_type: VOTER last_known_addr { host: "127.1.181.252" port: 44487 } } }
23:30:56.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:56.433219 1995 sys_catalog.cc:458] T 00000000000000000000000000000000 P 42441f8110d24b78a8ea9b73a71776d9 [sys.catalog]: This master's current role is: LEADER
23:30:56.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:56.442970 2010 sys_catalog.cc:455] T 00000000000000000000000000000000 P 726ed7c14b38409ca79aef03c750f165 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 42441f8110d24b78a8ea9b73a71776d9. Latest consensus state: current_term: 1 leader_uuid: "42441f8110d24b78a8ea9b73a71776d9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8624cf9c568e48ffa151e70ee3acd89a" member_type: VOTER last_known_addr { host: "127.1.181.254" port: 36363 } } peers { permanent_uuid: "726ed7c14b38409ca79aef03c750f165" member_type: VOTER last_known_addr { host: "127.1.181.253" port: 46667 } } peers { permanent_uuid: "42441f8110d24b78a8ea9b73a71776d9" member_type: VOTER last_known_addr { host: "127.1.181.252" port: 44487 } } }
23:30:56.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:56.443892 2010 sys_catalog.cc:458] T 00000000000000000000000000000000 P 726ed7c14b38409ca79aef03c750f165 [sys.catalog]: This master's current role is: FOLLOWER
23:30:56.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:56.462529 2030 sys_catalog.cc:455] T 00000000000000000000000000000000 P 726ed7c14b38409ca79aef03c750f165 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "42441f8110d24b78a8ea9b73a71776d9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8624cf9c568e48ffa151e70ee3acd89a" member_type: VOTER last_known_addr { host: "127.1.181.254" port: 36363 } } peers { permanent_uuid: "726ed7c14b38409ca79aef03c750f165" member_type: VOTER last_known_addr { host: "127.1.181.253" port: 46667 } } peers { permanent_uuid: "42441f8110d24b78a8ea9b73a71776d9" member_type: VOTER last_known_addr { host: "127.1.181.252" port: 44487 } } }
23:30:56.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:56.463382 2030 sys_catalog.cc:458] T 00000000000000000000000000000000 P 726ed7c14b38409ca79aef03c750f165 [sys.catalog]: This master's current role is: FOLLOWER
23:30:56.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:56.478487 2024 catalog_manager.cc:1370] Generated new certificate authority record
23:30:56.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:56.486330 2024 catalog_manager.cc:1504] Loading token signing keys...
23:30:56.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:56.524462 2024 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 42441f8110d24b78a8ea9b73a71776d9: Generated new TSK 0
23:30:56.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:56.528986 2024 catalog_manager.cc:1514] Initializing in-progress tserver states...
23:30:56.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:56.548974 1751 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskpPnkEK/build/tsan/bin/kudu
23:30:56.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskpPnkEK/build/tsan/bin/kudu
23:30:56.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/ts-0/wal
23:30:56.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/ts-0/data
23:30:56.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:30:56.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:30:56.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:30:56.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:30:56.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:30:56.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:30:56.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/ts-0/logs
23:30:56.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/ts-0/data/info.pb
23:30:56.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:30:56.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:30:56.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:30:56.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:30:56.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:30:56.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:30:56.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:30:56.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:30:56.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:30:56.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:30:56.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.181.193:0
23:30:56.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.181.193
23:30:56.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.181.193
23:30:56.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:30:56.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.181.254:36363,127.1.181.253:46667,127.1.181.252:44487
23:30:56.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.181.212:42595
23:30:56.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:30:56.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:30:56.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:30:56.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:56.652696 1831 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.181.252" port: 44487 } attempt: 7
23:30:56.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:56.740458 1831 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8624cf9c568e48ffa151e70ee3acd89a: Bootstrap starting.
23:30:56.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:56.789541 1831 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 8624cf9c568e48ffa151e70ee3acd89a: Neither blocks nor log segments found. Creating new log.
23:30:56.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:56.804610 1831 log.cc:824] T 00000000000000000000000000000000 P 8624cf9c568e48ffa151e70ee3acd89a: Log is configured to *not* fsync() on all Append() calls
23:30:56.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:56.847229 1831 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8624cf9c568e48ffa151e70ee3acd89a: No bootstrap required, opened a new log
23:30:57.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:57.032685 1831 raft_consensus.cc:348] T 00000000000000000000000000000000 P 8624cf9c568e48ffa151e70ee3acd89a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8624cf9c568e48ffa151e70ee3acd89a" member_type: VOTER last_known_addr { host: "127.1.181.254" port: 36363 } } peers { permanent_uuid: "726ed7c14b38409ca79aef03c750f165" member_type: VOTER last_known_addr { host: "127.1.181.253" port: 46667 } } peers { permanent_uuid: "42441f8110d24b78a8ea9b73a71776d9" member_type: VOTER last_known_addr { host: "127.1.181.252" port: 44487 } }
23:30:57.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:57.038575 1831 raft_consensus.cc:374] T 00000000000000000000000000000000 P 8624cf9c568e48ffa151e70ee3acd89a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
23:30:57.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:57.040391 1831 raft_consensus.cc:729] T 00000000000000000000000000000000 P 8624cf9c568e48ffa151e70ee3acd89a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8624cf9c568e48ffa151e70ee3acd89a, State: Initialized, Role: FOLLOWER
23:30:57.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:57.046164 1831 consensus_queue.cc:260] T 00000000000000000000000000000000 P 8624cf9c568e48ffa151e70ee3acd89a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8624cf9c568e48ffa151e70ee3acd89a" member_type: VOTER last_known_addr { host: "127.1.181.254" port: 36363 } } peers { permanent_uuid: "726ed7c14b38409ca79aef03c750f165" member_type: VOTER last_known_addr { host: "127.1.181.253" port: 46667 } } peers { permanent_uuid: "42441f8110d24b78a8ea9b73a71776d9" member_type: VOTER last_known_addr { host: "127.1.181.252" port: 44487 } }
23:30:57.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:57.058522 2039 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8624cf9c568e48ffa151e70ee3acd89a [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8624cf9c568e48ffa151e70ee3acd89a" member_type: VOTER last_known_addr { host: "127.1.181.254" port: 36363 } } peers { permanent_uuid: "726ed7c14b38409ca79aef03c750f165" member_type: VOTER last_known_addr { host: "127.1.181.253" port: 46667 } } peers { permanent_uuid: "42441f8110d24b78a8ea9b73a71776d9" member_type: VOTER last_known_addr { host: "127.1.181.252" port: 44487 } } }
23:30:57.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:57.059224 2039 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8624cf9c568e48ffa151e70ee3acd89a [sys.catalog]: This master's current role is: FOLLOWER
23:30:57.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:57.067971 1806 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "42441f8110d24b78a8ea9b73a71776d9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8624cf9c568e48ffa151e70ee3acd89a" is_pre_election: true
23:30:57.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:57.068781 1806 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 8624cf9c568e48ffa151e70ee3acd89a [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 42441f8110d24b78a8ea9b73a71776d9 in term 0.
23:30:57.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:57.070199 1805 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "42441f8110d24b78a8ea9b73a71776d9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8624cf9c568e48ffa151e70ee3acd89a"
23:30:57.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:57.070770 1805 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 8624cf9c568e48ffa151e70ee3acd89a [term 0 FOLLOWER]: Advancing to term 1
23:30:57.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:57.077656 1831 sys_catalog.cc:564] T 00000000000000000000000000000000 P 8624cf9c568e48ffa151e70ee3acd89a [sys.catalog]: configured and running, proceeding with master startup.
23:30:57.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:57.078558 1805 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 8624cf9c568e48ffa151e70ee3acd89a [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 42441f8110d24b78a8ea9b73a71776d9 in term 1.
23:30:57.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:57.079818 1804 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 8624cf9c568e48ffa151e70ee3acd89a [term 1 FOLLOWER]: Refusing update from remote peer 42441f8110d24b78a8ea9b73a71776d9: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
23:30:57.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:57.087278 2040 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 42441f8110d24b78a8ea9b73a71776d9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8624cf9c568e48ffa151e70ee3acd89a" member_type: VOTER last_known_addr { host: "127.1.181.254" port: 36363 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
23:30:57.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:57.094466 2032 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:30:57.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:57.095527 2032 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:30:57.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:57.096289 2032 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:30:57.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:57.169248 2047 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8624cf9c568e48ffa151e70ee3acd89a [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "42441f8110d24b78a8ea9b73a71776d9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8624cf9c568e48ffa151e70ee3acd89a" member_type: VOTER last_known_addr { host: "127.1.181.254" port: 36363 } } peers { permanent_uuid: "726ed7c14b38409ca79aef03c750f165" member_type: VOTER last_known_addr { host: "127.1.181.253" port: 46667 } } peers { permanent_uuid: "42441f8110d24b78a8ea9b73a71776d9" member_type: VOTER last_known_addr { host: "127.1.181.252" port: 44487 } } }
23:30:57.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:57.169144 2039 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8624cf9c568e48ffa151e70ee3acd89a [sys.catalog]: SysCatalogTable state changed. Reason: New leader 42441f8110d24b78a8ea9b73a71776d9. Latest consensus state: current_term: 1 leader_uuid: "42441f8110d24b78a8ea9b73a71776d9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8624cf9c568e48ffa151e70ee3acd89a" member_type: VOTER last_known_addr { host: "127.1.181.254" port: 36363 } } peers { permanent_uuid: "726ed7c14b38409ca79aef03c750f165" member_type: VOTER last_known_addr { host: "127.1.181.253" port: 46667 } } peers { permanent_uuid: "42441f8110d24b78a8ea9b73a71776d9" member_type: VOTER last_known_addr { host: "127.1.181.252" port: 44487 } } }
23:30:57.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:57.170022 2047 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8624cf9c568e48ffa151e70ee3acd89a [sys.catalog]: This master's current role is: FOLLOWER
23:30:57.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:57.170130 2039 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8624cf9c568e48ffa151e70ee3acd89a [sys.catalog]: This master's current role is: FOLLOWER
23:30:57.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:57.196841 2032 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:30:57.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:57.198300 2032 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.181.193
23:30:57.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:57.230119 2056 catalog_manager.cc:1259] Loaded cluster ID: b175d83e22ab431c8acddd74bc0c42a2
23:30:57.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:57.230387 2056 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 8624cf9c568e48ffa151e70ee3acd89a: loading cluster ID for follower catalog manager: success
23:30:57.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:57.250175 2056 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 8624cf9c568e48ffa151e70ee3acd89a: acquiring CA information for follower catalog manager: success
23:30:57.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:57.255486 2056 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 8624cf9c568e48ffa151e70ee3acd89a: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
23:30:57.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:57.268050 2032 tablet_server_runner.cc:78] Tablet server non-default flags:
23:30:57.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:30:57.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.181.212:42595
23:30:57.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:30:57.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:30:57.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/ts-0/data
23:30:57.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/ts-0/wal
23:30:57.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:30:57.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:30:57.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.181.193:0
23:30:57.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:30:57.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:30:57.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:30:57.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/ts-0/data/info.pb
23:30:57.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.181.193
23:30:57.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:30:57.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.181.254:36363,127.1.181.253:46667,127.1.181.252:44487
23:30:57.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:30:57.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:30:57.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:30:57.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:30:57.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:30:57.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.181.193
23:30:57.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/ts-0/logs
23:30:57.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:30:57.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:30:57.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
23:30:57.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:30:57.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:30:57.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:30:57.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:30:57.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 22:42:10 UTC on ccycloud.quasar-yawxpd.root.comops.site
23:30:57.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676391
23:30:57.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:30:57.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:57.269495 2032 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:30:57.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:57.271597 2032 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:30:57.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:57.314373 2060 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:30:57.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:57.354954 2023 catalog_manager.cc:1259] Loaded cluster ID: b175d83e22ab431c8acddd74bc0c42a2
23:30:57.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:57.355741 2023 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 726ed7c14b38409ca79aef03c750f165: loading cluster ID for follower catalog manager: success
23:30:57.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:57.378587 2023 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 726ed7c14b38409ca79aef03c750f165: acquiring CA information for follower catalog manager: success
23:30:57.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:57.402077 2023 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 726ed7c14b38409ca79aef03c750f165: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
23:30:57.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:57.620345 2040 sys_catalog.cc:455] T 00000000000000000000000000000000 P 42441f8110d24b78a8ea9b73a71776d9 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "42441f8110d24b78a8ea9b73a71776d9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8624cf9c568e48ffa151e70ee3acd89a" member_type: VOTER last_known_addr { host: "127.1.181.254" port: 36363 } } peers { permanent_uuid: "726ed7c14b38409ca79aef03c750f165" member_type: VOTER last_known_addr { host: "127.1.181.253" port: 46667 } } peers { permanent_uuid: "42441f8110d24b78a8ea9b73a71776d9" member_type: VOTER last_known_addr { host: "127.1.181.252" port: 44487 } } }
23:30:57.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:57.621924 2040 sys_catalog.cc:458] T 00000000000000000000000000000000 P 42441f8110d24b78a8ea9b73a71776d9 [sys.catalog]: This master's current role is: LEADER
23:30:59.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:57.318136 2059 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:30:59.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:59.470875 2032 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 2.162s user 0.590s sys 0.911s
23:30:59.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:58.695842 2058 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 2032
23:30:59.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:59.471293 2032 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 2.162s user 0.590s sys 0.911s
23:30:59.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:59.471570 2032 server_base.cc:1034] running on GCE node
23:30:59.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:30:59.473169 2062 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:30:59.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:59.476799 2032 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:30:59.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:59.479642 2032 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:30:59.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:59.481096 2032 hybrid_clock.cc:648] HybridClock initialized: now 1790551859480829 us; error 263 us; skew 500 ppm
23:30:59.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:59.482247 2032 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:30:59.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:59.494727 2032 webserver.cc:460] Webserver started at http://127.1.181.193:35015/ using document root <none> and password file <none>
23:30:59.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:59.496031 2032 fs_manager.cc:359] Metadata directory not provided
23:30:59.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:59.496306 2032 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:30:59.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:59.496922 2032 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
23:30:59.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:59.504334 2032 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/ts-0/data/instance:
23:30:59.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "29db0b53dda24388bf262750a3286a43"
23:30:59.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 23:30:59 on dist-test-slave-dnlc"
23:30:59.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:59.505884 2032 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/ts-0/wal/instance:
23:30:59.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "29db0b53dda24388bf262750a3286a43"
23:30:59.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 23:30:59 on dist-test-slave-dnlc"
23:30:59.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:59.515471 2032 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.004s sys 0.004s
23:30:59.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:59.523607 2075 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
23:30:59.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:59.524888 2032 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.001s
23:30:59.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:59.525257 2032 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/ts-0/data,/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/ts-0/wal
23:30:59.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "29db0b53dda24388bf262750a3286a43"
23:30:59.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 23:30:59 on dist-test-slave-dnlc"
23:30:59.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:59.525765 2032 fs_report.cc:383] FS layout report
23:30:59.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:30:59.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/ts-0/wal
23:30:59.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/ts-0/wal
23:30:59.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/ts-0/data/data
23:30:59.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:30:59.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:30:59.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:30:59.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:30:59.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:30:59.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:30:59.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:30:59.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:30:59.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:30:59.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:30:59.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:30:59.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:59.581120 2032 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
23:30:59.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:59.583076 2032 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:30:59.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:59.583633 2032 kserver.cc:163] Server-wide thread pool size limit: 3276
23:30:59.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:59.587294 2032 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
23:30:59.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:59.592847 2032 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
23:30:59.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:59.593132 2032 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
23:30:59.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:59.593433 2032 ts_tablet_manager.cc:610] Registered 0 tablets
23:30:59.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:59.593667 2032 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.001s sys 0.000s
23:30:59.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:59.769333 2032 rpc_server.cc:307] RPC server started. Bound to: 127.1.181.193:43237
23:30:59.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:59.769505 2187 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.181.193:43237 every 8 connection(s)
23:30:59.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:59.772001 2032 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/ts-0/data/info.pb
23:30:59.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:59.781193 1751 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskpPnkEK/build/tsan/bin/kudu as pid 2032
23:30:59.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:59.781760 1751 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/ts-0/wal/instance
23:30:59.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:59.800081 1751 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskpPnkEK/build/tsan/bin/kudu
23:30:59.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskpPnkEK/build/tsan/bin/kudu
23:30:59.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/ts-1/wal
23:30:59.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/ts-1/data
23:30:59.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:30:59.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:30:59.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:30:59.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:30:59.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:30:59.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:30:59.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/ts-1/logs
23:30:59.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/ts-1/data/info.pb
23:30:59.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:30:59.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:30:59.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:30:59.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:30:59.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:30:59.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:30:59.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:30:59.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:30:59.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:30:59.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:30:59.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.181.194:0
23:30:59.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.181.194
23:30:59.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.181.194
23:30:59.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:30:59.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.181.254:36363,127.1.181.253:46667,127.1.181.252:44487
23:30:59.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.181.212:42595
23:30:59.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:30:59.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:30:59.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:30:59.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:59.818336 2188 heartbeater.cc:347] Connected to a master server at 127.1.181.252:44487
23:30:59.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:59.819434 2188 heartbeater.cc:464] Registering TS with master...
23:30:59.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:59.821667 2188 heartbeater.cc:511] Master 127.1.181.252:44487 requested a full tablet report, sending...
23:30:59.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:59.825119 1949 ts_manager.cc:194] Registered new tserver with Master: 29db0b53dda24388bf262750a3286a43 (127.1.181.193:43237)
23:30:59.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:59.828460 1949 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.1.181.193:56431
23:30:59.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:59.841231 2189 heartbeater.cc:347] Connected to a master server at 127.1.181.253:46667
23:30:59.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:59.841643 2189 heartbeater.cc:464] Registering TS with master...
23:30:59.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:59.842459 2189 heartbeater.cc:511] Master 127.1.181.253:46667 requested a full tablet report, sending...
23:30:59.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:59.845657 1871 ts_manager.cc:194] Registered new tserver with Master: 29db0b53dda24388bf262750a3286a43 (127.1.181.193:43237)
23:30:59.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:59.854066 2190 heartbeater.cc:347] Connected to a master server at 127.1.181.254:36363
23:30:59.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:59.854405 2190 heartbeater.cc:464] Registering TS with master...
23:30:59.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:59.855162 2190 heartbeater.cc:511] Master 127.1.181.254:36363 requested a full tablet report, sending...
23:30:59.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:30:59.858209 1796 ts_manager.cc:194] Registered new tserver with Master: 29db0b53dda24388bf262750a3286a43 (127.1.181.193:43237)
23:31:00.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:00.159621 2195 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:31:00.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:00.160058 2195 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:31:00.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:00.160446 2195 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:31:00.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:00.190677 2195 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:31:00.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:00.191531 2195 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.181.194
23:31:00.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:00.224391 2195 tablet_server_runner.cc:78] Tablet server non-default flags:
23:31:00.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:31:00.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.181.212:42595
23:31:00.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:31:00.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:31:00.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/ts-1/data
23:31:00.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/ts-1/wal
23:31:00.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:31:00.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:31:00.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.181.194:0
23:31:00.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:31:00.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:31:00.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:31:00.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/ts-1/data/info.pb
23:31:00.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.181.194
23:31:00.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:31:00.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.181.254:36363,127.1.181.253:46667,127.1.181.252:44487
23:31:00.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:31:00.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:31:00.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:31:00.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:31:00.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:31:00.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.181.194
23:31:00.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/ts-1/logs
23:31:00.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:31:00.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:31:00.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
23:31:00.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:31:00.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:31:00.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:31:00.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:31:00.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 22:42:10 UTC on ccycloud.quasar-yawxpd.root.comops.site
23:31:00.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676391
23:31:00.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:31:00.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:00.225502 2195 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:31:00.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:00.227198 2195 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:31:00.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:00.249819 2205 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:31:00.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:00.833202 2188 heartbeater.cc:503] Master 127.1.181.252:44487 was elected leader, sending a full tablet report...
23:31:01.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:00.250458 2204 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:31:01.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:01.444948 2206 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
23:31:01.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:00.250341 2207 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:31:01.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:01.445542 2195 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
23:31:01.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:01.448654 2195 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:31:01.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:01.451388 2195 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:31:01.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:01.452821 2195 hybrid_clock.cc:648] HybridClock initialized: now 1790551861452785 us; error 50 us; skew 500 ppm
23:31:01.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:01.453749 2195 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:31:01.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:01.460234 2195 webserver.cc:460] Webserver started at http://127.1.181.194:45403/ using document root <none> and password file <none>
23:31:01.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:01.461071 2195 fs_manager.cc:359] Metadata directory not provided
23:31:01.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:01.461248 2195 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:31:01.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:01.461673 2195 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
23:31:01.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:01.466665 2195 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/ts-1/data/instance:
23:31:01.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "77caaec5a1ee44c8a826abbe8eda79f5"
23:31:01.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 23:31:01 on dist-test-slave-dnlc"
23:31:01.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:01.467860 2195 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/ts-1/wal/instance:
23:31:01.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "77caaec5a1ee44c8a826abbe8eda79f5"
23:31:01.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 23:31:01 on dist-test-slave-dnlc"
23:31:01.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:01.478927 2195 fs_manager.cc:691] Time spent creating directory manager: real 0.010s user 0.010s sys 0.001s
23:31:01.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:01.484911 2216 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
23:31:01.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:01.485996 2195 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
23:31:01.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:01.486338 2195 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/ts-1/data,/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/ts-1/wal
23:31:01.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "77caaec5a1ee44c8a826abbe8eda79f5"
23:31:01.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 23:31:01 on dist-test-slave-dnlc"
23:31:01.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:01.486711 2195 fs_report.cc:383] FS layout report
23:31:01.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:31:01.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/ts-1/wal
23:31:01.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/ts-1/wal
23:31:01.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/ts-1/data/data
23:31:01.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:31:01.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:31:01.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:31:01.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:31:01.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:31:01.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:31:01.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:31:01.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:31:01.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:31:01.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:31:01.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:31:01.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:01.533926 2195 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
23:31:01.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:01.535539 2195 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:31:01.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:01.536068 2195 kserver.cc:163] Server-wide thread pool size limit: 3276
23:31:01.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:01.538516 2195 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
23:31:01.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:01.542521 2195 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
23:31:01.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:01.542768 2195 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
23:31:01.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:01.543075 2195 ts_tablet_manager.cc:610] Registered 0 tablets
23:31:01.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:01.543267 2195 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
23:31:01.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:01.695668 2195 rpc_server.cc:307] RPC server started. Bound to: 127.1.181.194:46017
23:31:01.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:01.695765 2328 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.181.194:46017 every 8 connection(s)
23:31:01.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:01.698283 2195 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/ts-1/data/info.pb
23:31:01.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:01.708051 1751 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskpPnkEK/build/tsan/bin/kudu as pid 2195
23:31:01.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:01.708859 1751 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/ts-1/wal/instance
23:31:01.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:01.726743 1751 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskpPnkEK/build/tsan/bin/kudu
23:31:01.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskpPnkEK/build/tsan/bin/kudu
23:31:01.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/ts-2/wal
23:31:01.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/ts-2/data
23:31:01.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:31:01.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:31:01.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:31:01.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:31:01.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:31:01.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:31:01.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/ts-2/logs
23:31:01.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/ts-2/data/info.pb
23:31:01.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:31:01.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:31:01.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:31:01.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:31:01.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:31:01.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:31:01.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:31:01.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:31:01.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:31:01.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:31:01.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.181.195:0
23:31:01.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.181.195
23:31:01.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.181.195
23:31:01.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:31:01.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.181.254:36363,127.1.181.253:46667,127.1.181.252:44487
23:31:01.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.181.212:42595
23:31:01.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:31:01.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:31:01.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:31:01.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:01.742065 2329 heartbeater.cc:347] Connected to a master server at 127.1.181.252:44487
23:31:01.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:01.742553 2329 heartbeater.cc:464] Registering TS with master...
23:31:01.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:01.744079 2329 heartbeater.cc:511] Master 127.1.181.252:44487 requested a full tablet report, sending...
23:31:01.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:01.746206 2330 heartbeater.cc:347] Connected to a master server at 127.1.181.253:46667
23:31:01.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:01.746523 2330 heartbeater.cc:464] Registering TS with master...
23:31:01.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:01.746765 1949 ts_manager.cc:194] Registered new tserver with Master: 77caaec5a1ee44c8a826abbe8eda79f5 (127.1.181.194:46017)
23:31:01.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:01.747275 2330 heartbeater.cc:511] Master 127.1.181.253:46667 requested a full tablet report, sending...
23:31:01.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:01.749125 1949 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.1.181.194:34427
23:31:01.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:01.750279 2331 heartbeater.cc:347] Connected to a master server at 127.1.181.254:36363
23:31:01.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:01.750218 1871 ts_manager.cc:194] Registered new tserver with Master: 77caaec5a1ee44c8a826abbe8eda79f5 (127.1.181.194:46017)
23:31:01.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:01.750576 2331 heartbeater.cc:464] Registering TS with master...
23:31:01.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:01.751295 2331 heartbeater.cc:511] Master 127.1.181.254:36363 requested a full tablet report, sending...
23:31:01.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:01.754269 1796 ts_manager.cc:194] Registered new tserver with Master: 77caaec5a1ee44c8a826abbe8eda79f5 (127.1.181.194:46017)
23:31:02.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:02.067364 2339 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:31:02.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:02.067818 2339 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:31:02.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:02.068441 2339 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:31:02.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:02.099993 2339 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:31:02.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:02.100775 2339 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.181.195
23:31:02.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:02.133895 2339 tablet_server_runner.cc:78] Tablet server non-default flags:
23:31:02.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:31:02.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.181.212:42595
23:31:02.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:31:02.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:31:02.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/ts-2/data
23:31:02.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/ts-2/wal
23:31:02.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:31:02.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:31:02.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.181.195:0
23:31:02.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:31:02.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:31:02.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:31:02.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/ts-2/data/info.pb
23:31:02.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.181.195
23:31:02.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:31:02.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.181.254:36363,127.1.181.253:46667,127.1.181.252:44487
23:31:02.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:31:02.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:31:02.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:31:02.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:31:02.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:31:02.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.181.195
23:31:02.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/ts-2/logs
23:31:02.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:31:02.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:31:02.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
23:31:02.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:31:02.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:31:02.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:31:02.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:31:02.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 22:42:10 UTC on ccycloud.quasar-yawxpd.root.comops.site
23:31:02.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676391
23:31:02.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:31:02.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:02.135213 2339 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:31:02.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:02.136899 2339 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:31:02.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:02.155020 2345 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:31:02.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:02.754490 2329 heartbeater.cc:503] Master 127.1.181.252:44487 was elected leader, sending a full tablet report...
23:31:03.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:02.156461 2348 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:31:03.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:02.156774 2339 server_base.cc:1034] running on GCE node
23:31:03.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:02.155055 2346 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:31:03.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:03.355034 2339 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:31:03.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:03.357672 2339 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:31:03.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:03.359066 2339 hybrid_clock.cc:648] HybridClock initialized: now 1790551863359018 us; error 60 us; skew 500 ppm
23:31:03.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:03.359805 2339 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:31:03.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:03.366569 2339 webserver.cc:460] Webserver started at http://127.1.181.195:34549/ using document root <none> and password file <none>
23:31:03.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:03.367483 2339 fs_manager.cc:359] Metadata directory not provided
23:31:03.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:03.367664 2339 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:31:03.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:03.368054 2339 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
23:31:03.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:03.372848 2339 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/ts-2/data/instance:
23:31:03.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "434b87930a3a4e6b9b216ec220460f7f"
23:31:03.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 23:31:03 on dist-test-slave-dnlc"
23:31:03.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:03.373932 2339 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/ts-2/wal/instance:
23:31:03.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "434b87930a3a4e6b9b216ec220460f7f"
23:31:03.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 23:31:03 on dist-test-slave-dnlc"
23:31:03.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:03.381331 2339 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.003s sys 0.004s
23:31:03.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:03.386617 2356 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
23:31:03.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:03.387663 2339 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.000s sys 0.005s
23:31:03.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:03.387974 2339 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/ts-2/data,/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/ts-2/wal
23:31:03.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "434b87930a3a4e6b9b216ec220460f7f"
23:31:03.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 23:31:03 on dist-test-slave-dnlc"
23:31:03.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:03.388278 2339 fs_report.cc:383] FS layout report
23:31:03.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:31:03.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/ts-2/wal
23:31:03.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/ts-2/wal
23:31:03.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/ts-2/data/data
23:31:03.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:31:03.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:31:03.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:31:03.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:31:03.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:31:03.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:31:03.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:31:03.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:31:03.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:31:03.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:31:03.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:31:03.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:03.435159 2339 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
23:31:03.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:03.436638 2339 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:31:03.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:03.437068 2339 kserver.cc:163] Server-wide thread pool size limit: 3276
23:31:03.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:03.439548 2339 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
23:31:03.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:03.443624 2339 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
23:31:03.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:03.443825 2339 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
23:31:03.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:03.444037 2339 ts_tablet_manager.cc:610] Registered 0 tablets
23:31:03.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:03.444238 2339 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
23:31:03.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:03.596444 2339 rpc_server.cc:307] RPC server started. Bound to: 127.1.181.195:33283
23:31:03.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:03.596539 2468 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.181.195:33283 every 8 connection(s)
23:31:03.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:03.598963 2339 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/ts-2/data/info.pb
23:31:03.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:03.603618 1751 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskpPnkEK/build/tsan/bin/kudu as pid 2339
23:31:03.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:03.604111 1751 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster8777079865774462076/ts-2/wal/instance
23:31:03.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:03.652122 2470 heartbeater.cc:347] Connected to a master server at 127.1.181.253:46667
23:31:03.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:03.652596 2470 heartbeater.cc:464] Registering TS with master...
23:31:03.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:03.653699 2470 heartbeater.cc:511] Master 127.1.181.253:46667 requested a full tablet report, sending...
23:31:03.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:03.657440 1871 ts_manager.cc:194] Registered new tserver with Master: 434b87930a3a4e6b9b216ec220460f7f (127.1.181.195:33283)
23:31:03.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:03.677464 2474 heartbeater.cc:347] Connected to a master server at 127.1.181.254:36363
23:31:03.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:03.677848 2474 heartbeater.cc:464] Registering TS with master...
23:31:03.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:03.678750 2474 heartbeater.cc:511] Master 127.1.181.254:36363 requested a full tablet report, sending...
23:31:03.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:03.681244 1796 ts_manager.cc:194] Registered new tserver with Master: 434b87930a3a4e6b9b216ec220460f7f (127.1.181.195:33283)
23:31:03.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:03.682034 2469 heartbeater.cc:347] Connected to a master server at 127.1.181.252:44487
23:31:03.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:03.682360 2469 heartbeater.cc:464] Registering TS with master...
23:31:03.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:03.683018 2469 heartbeater.cc:511] Master 127.1.181.252:44487 requested a full tablet report, sending...
23:31:03.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:03.684854 1949 ts_manager.cc:194] Registered new tserver with Master: 434b87930a3a4e6b9b216ec220460f7f (127.1.181.195:33283)
23:31:03.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:03.687292 1949 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.1.181.195:54019
23:31:03.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:03.688530 1751 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
23:31:03.701 [DEBUG - main] (MiniKuduCluster.java:195) Response:
23:31:03.702 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
23:31:03.715 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.1.181.254"
port: 36363
}
bound_http_address {
host: "127.1.181.254"
port: 33993
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.1.181.253"
port: 46667
}
bound_http_address {
host: "127.1.181.253"
port: 41099
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.1.181.252"
port: 44487
}
bound_http_address {
host: "127.1.181.252"
port: 40067
}
}
}
23:31:03.721 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
23:31:03.723 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.1.181.193"
port: 43237
}
bound_http_address {
host: "127.1.181.193"
port: 35015
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.1.181.194"
port: 46017
}
bound_http_address {
host: "127.1.181.194"
port: 45403
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.1.181.195"
port: 33283
}
bound_http_address {
host: "127.1.181.195"
port: 34549
}
}
}
23:31:03.723 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
23:31:03.901 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
23:31:04.007 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskpPnkEK/test-tmp
23:31:04.008 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
23:31:04.008 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskpPnkEK/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
23:31:04.015 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
23:31:04.334 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
23:31:04.334 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
23:31:04.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:04.336855 2480 mini_oidc.cc:152] Starting JWKS server
23:31:04.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:04.340523 2480 webserver.cc:460] Webserver started at http://127.0.0.1:44517/ using document root <none> and password file <none>
23:31:04.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:04.340904 2480 mini_oidc.cc:189] Starting OIDC Discovery server
23:31:04.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:04.342614 2480 webserver.cc:460] Webserver started at http://127.0.0.1:42963/ using document root <none> and password file <none>
23:31:04.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
23:31:04.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/krb5kdc/principal' for realm 'KRBTEST.COM',
23:31:04.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
23:31:04.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 23:31:04 dist-test-slave-dnlc krb5kdc[2493](info): setting up network...
23:31:04.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
23:31:04.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 23:31:04 dist-test-slave-dnlc krb5kdc[2493](info): set up 2 sockets
23:31:04.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 23:31:04 dist-test-slave-dnlc krb5kdc[2493](info): commencing operation
23:31:04.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
23:31:04.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:04.690557 2469 heartbeater.cc:503] Master 127.1.181.252:44487 was elected leader, sending a full tablet report...
23:31:05.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:05.447050 2480 mini_kdc.cc:121] Time spent starting KDC: real 1.080s user 0.000s sys 0.020s
23:31:05.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
23:31:05.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
23:31:05.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
23:31:05.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
23:31:05.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
23:31:05.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
23:31:05.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
23:31:05.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
23:31:05.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
23:31:05.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
23:31:05.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/krb5kdc/test-user.keytab.
23:31:05.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/krb5kdc/test-user.keytab.
23:31:05.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 23:31:05 dist-test-slave-dnlc krb5kdc[2493](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790551865, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
23:31:05.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
23:31:05.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T23:31:05Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
23:31:05.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T23:31:05Z Disabled control of system clock
23:31:05.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.2.108.62@KRBTEST.COM; defaulting to no policy
23:31:05.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
23:31:05.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.2.108.62@KRBTEST.COM" created.
23:31:05.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
23:31:05.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.108.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/krb5kdc/kudu_127.2.108.62.keytab.
23:31:05.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.108.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/krb5kdc/kudu_127.2.108.62.keytab.
23:31:05.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:05.638374 2480 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskpPnkEK/build/tsan/bin/kudu
23:31:05.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskpPnkEK/build/tsan/bin/kudu
23:31:05.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-0/wal
23:31:05.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-0/data
23:31:05.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:31:05.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:31:05.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:31:05.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:31:05.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:31:05.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:31:05.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-0/logs
23:31:05.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-0/data/info.pb
23:31:05.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:31:05.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:31:05.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:31:05.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:31:05.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:31:05.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:31:05.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:31:05.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:31:05.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
23:31:05.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:31:05.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:31:05.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:31:05.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.108.62:39471
23:31:05.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.108.62
23:31:05.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:31:05.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/krb5kdc/kudu_127.2.108.62.keytab
23:31:05.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.108.62
23:31:05.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
23:31:05.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
23:31:05.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
23:31:05.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.108.20:43467
23:31:05.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:31:05.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:31:05.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:31:05.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:31:05.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.108.62:39471,127.2.108.61:33947,127.2.108.60:42421
23:31:05.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:31:05.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44517/jwks.json
23:31:05.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
23:31:05.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:05.959586 2506 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:31:05.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:05.960155 2506 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:31:05.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:05.960709 2506 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:31:05.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:05.991972 2506 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
23:31:05.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:05.992198 2506 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
23:31:05.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:05.992458 2506 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:31:05.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:05.992650 2506 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
23:31:05.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:05.992832 2506 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
23:31:05.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:05.993026 2506 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
23:31:05.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:05.993162 2506 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44517/jwks.json
23:31:06.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:06.039294 2506 master_runner.cc:386] Master server non-default flags:
23:31:06.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:31:06.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.108.20:43467
23:31:06.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:31:06.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:31:06.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-0/data
23:31:06.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-0/wal
23:31:06.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
23:31:06.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:31:06.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.108.62:39471,127.2.108.61:33947,127.2.108.60:42421
23:31:06.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/krb5kdc/kudu_127.2.108.62.keytab
23:31:06.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.108.62
23:31:06.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:31:06.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:31:06.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:31:06.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.108.62:39471
23:31:06.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:31:06.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:31:06.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:31:06.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44517/jwks.json
23:31:06.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
23:31:06.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
23:31:06.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
23:31:06.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:31:06.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:31:06.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-0/data/info.pb
23:31:06.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.108.62
23:31:06.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:31:06.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:31:06.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:31:06.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:31:06.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:31:06.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:31:06.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-0/logs
23:31:06.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:31:06.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:31:06.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
23:31:06.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
23:31:06.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:31:06.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:31:06.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:31:06.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 22:42:10 UTC on ccycloud.quasar-yawxpd.root.comops.site
23:31:06.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676391
23:31:06.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:31:06.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:06.040436 2506 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:31:06.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:06.041322 2506 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:31:06.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:06.059305 2513 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:31:07.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:06.059597 2512 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:31:07.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:06.059309 2515 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:31:07.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:06.061340 2506 server_base.cc:1034] running on GCE node
23:31:07.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:07.268774 2506 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:31:07.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:07.271845 2506 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:31:07.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:07.273269 2506 hybrid_clock.cc:648] HybridClock initialized: now 1790551867273209 us; error 73 us; skew 500 ppm
23:31:07.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:07.274109 2506 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:31:07.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 23:31:07 dist-test-slave-dnlc krb5kdc[2493](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790551867, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.108.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
23:31:07.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:07.285269 2506 init.cc:377] Logged in from keytab as kudu/127.2.108.62@KRBTEST.COM (short username kudu)
23:31:07.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:07.291375 2506 webserver.cc:460] Webserver started at http://127.2.108.62:34899/ using document root <none> and password file <none>
23:31:07.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:07.292336 2506 fs_manager.cc:359] Metadata directory not provided
23:31:07.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:07.292568 2506 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:31:07.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:07.293057 2506 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
23:31:07.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:07.297727 2506 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-0/data/instance:
23:31:07.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f6cda9ea914a448ebe7d4c25dec6364f"
23:31:07.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 23:31:07 on dist-test-slave-dnlc"
23:31:07.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:07.298875 2506 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-0/wal/instance:
23:31:07.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f6cda9ea914a448ebe7d4c25dec6364f"
23:31:07.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 23:31:07 on dist-test-slave-dnlc"
23:31:07.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:07.306603 2506 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
23:31:07.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:07.312758 2524 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
23:31:07.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:07.313954 2506 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.006s sys 0.001s
23:31:07.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:07.314215 2506 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-0/data,/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-0/wal
23:31:07.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f6cda9ea914a448ebe7d4c25dec6364f"
23:31:07.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 23:31:07 on dist-test-slave-dnlc"
23:31:07.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:07.314505 2506 fs_report.cc:383] FS layout report
23:31:07.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:31:07.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-0/wal
23:31:07.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-0/wal
23:31:07.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-0/data/data
23:31:07.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:31:07.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:31:07.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:31:07.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:31:07.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:31:07.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:31:07.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:31:07.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:31:07.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:31:07.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:31:07.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:31:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:07.355011 2506 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
23:31:07.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:07.360363 2506 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:31:07.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:07.361112 2506 kserver.cc:163] Server-wide thread pool size limit: 3276
23:31:07.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:07.436946 2506 rpc_server.cc:307] RPC server started. Bound to: 127.2.108.62:39471
23:31:07.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:07.437139 2575 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.2.108.62:39471 every 8 connection(s)
23:31:07.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:07.439623 2506 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-0/data/info.pb
23:31:07.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:07.445065 2576 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
23:31:07.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:07.448088 2480 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskpPnkEK/build/tsan/bin/kudu as pid 2506
23:31:07.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:07.448619 2480 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-0/wal/instance
23:31:07.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:07.461539 2576 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.108.62" port: 39471 } has no permanent_uuid. Determining permanent_uuid...
23:31:07.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.2.108.61@KRBTEST.COM; defaulting to no policy
23:31:07.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
23:31:07.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.2.108.61@KRBTEST.COM" created.
23:31:07.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 23:31:07 dist-test-slave-dnlc krb5kdc[2493](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790551867, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.108.62@KRBTEST.COM for kudu/127.2.108.62@KRBTEST.COM
23:31:07.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
23:31:07.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.108.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/krb5kdc/kudu_127.2.108.61.keytab.
23:31:07.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.108.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/krb5kdc/kudu_127.2.108.61.keytab.
23:31:07.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:07.500247 2480 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskpPnkEK/build/tsan/bin/kudu
23:31:07.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskpPnkEK/build/tsan/bin/kudu
23:31:07.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-1/wal
23:31:07.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-1/data
23:31:07.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:31:07.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:31:07.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:31:07.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:31:07.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:31:07.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:31:07.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-1/logs
23:31:07.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-1/data/info.pb
23:31:07.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:31:07.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:31:07.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:31:07.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:31:07.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:31:07.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:31:07.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:31:07.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:31:07.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
23:31:07.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:31:07.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:31:07.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:31:07.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.108.61:33947
23:31:07.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.108.61
23:31:07.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:31:07.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/krb5kdc/kudu_127.2.108.61.keytab
23:31:07.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.108.61
23:31:07.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
23:31:07.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
23:31:07.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
23:31:07.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.108.20:43467
23:31:07.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:31:07.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:31:07.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:31:07.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:31:07.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.108.62:39471,127.2.108.61:33947,127.2.108.60:42421
23:31:07.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:31:07.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44517/jwks.json
23:31:07.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
23:31:07.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:07.502288 2576 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.108.61" port: 33947 } has no permanent_uuid. Determining permanent_uuid...
23:31:07.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:07.504787 2526 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.108.61:33947: connect: Connection refused (error 111)
23:31:07.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:07.509068 2576 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.108.61:33947: Network error: Client connection negotiation failed: client connection to 127.2.108.61:33947: connect: Connection refused (error 111)
23:31:07.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:07.558511 2576 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.108.61" port: 33947 } attempt: 1
23:31:07.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:07.562103 2576 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.108.61:33947: Network error: Client connection negotiation failed: client connection to 127.2.108.61:33947: connect: Connection refused (error 111)
23:31:07.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:07.630545 2576 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.108.61" port: 33947 } attempt: 2
23:31:07.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:07.634686 2576 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.108.61:33947: Network error: Client connection negotiation failed: client connection to 127.2.108.61:33947: connect: Connection refused (error 111)
23:31:07.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:07.726105 2576 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.108.61" port: 33947 } attempt: 3
23:31:07.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:07.729830 2576 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.108.61:33947: Network error: Client connection negotiation failed: client connection to 127.2.108.61:33947: connect: Connection refused (error 111)
23:31:07.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:07.808542 2582 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:31:07.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:07.809208 2582 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:31:07.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:07.809809 2582 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:31:07.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:07.840503 2582 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
23:31:07.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:07.840711 2582 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
23:31:07.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:07.840969 2582 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:31:07.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:07.841152 2582 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
23:31:07.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:07.841403 2582 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
23:31:07.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:07.841596 2582 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
23:31:07.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:07.841749 2582 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44517/jwks.json
23:31:07.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:07.873243 2576 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.108.61" port: 33947 } attempt: 4
23:31:07.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:07.878199 2576 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.108.61:33947: Network error: Client connection negotiation failed: client connection to 127.2.108.61:33947: connect: Connection refused (error 111)
23:31:07.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:07.880980 2582 master_runner.cc:386] Master server non-default flags:
23:31:07.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:31:07.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.108.20:43467
23:31:07.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:31:07.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:31:07.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-1/data
23:31:07.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-1/wal
23:31:07.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
23:31:07.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:31:07.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.108.62:39471,127.2.108.61:33947,127.2.108.60:42421
23:31:07.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/krb5kdc/kudu_127.2.108.61.keytab
23:31:07.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.108.61
23:31:07.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:31:07.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:31:07.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:31:07.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.108.61:33947
23:31:07.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:31:07.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:31:07.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:31:07.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44517/jwks.json
23:31:07.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
23:31:07.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
23:31:07.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
23:31:07.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:31:07.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:31:07.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-1/data/info.pb
23:31:07.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.108.61
23:31:07.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:31:07.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:31:07.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:31:07.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:31:07.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:31:07.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:31:07.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-1/logs
23:31:07.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:31:07.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:31:07.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
23:31:07.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
23:31:07.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:31:07.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:31:07.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:31:07.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 22:42:10 UTC on ccycloud.quasar-yawxpd.root.comops.site
23:31:07.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676391
23:31:07.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:31:07.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:07.882179 2582 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:31:07.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:07.883489 2582 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:31:07.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:07.902153 2589 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:31:08.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:08.177603 2576 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.108.61" port: 33947 } attempt: 5
23:31:08.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:08.181974 2576 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.108.61:33947: Network error: Client connection negotiation failed: client connection to 127.2.108.61:33947: connect: Connection refused (error 111)
23:31:08.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:08.729351 2576 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.108.61" port: 33947 } attempt: 6
23:31:08.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:08.736888 2576 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.108.61:33947: Network error: Client connection negotiation failed: client connection to 127.2.108.61:33947: connect: Connection refused (error 111)
23:31:09.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:07.902520 2592 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:31:09.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:07.904165 2582 server_base.cc:1034] running on GCE node
23:31:09.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:07.902444 2590 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:31:09.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:09.122733 2582 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:31:09.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:09.125599 2582 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:31:09.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:09.127063 2582 hybrid_clock.cc:648] HybridClock initialized: now 1790551869127013 us; error 63 us; skew 500 ppm
23:31:09.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:09.127840 2582 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:31:09.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 23:31:09 dist-test-slave-dnlc krb5kdc[2493](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790551869, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.108.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
23:31:09.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:09.133507 2582 init.cc:377] Logged in from keytab as kudu/127.2.108.61@KRBTEST.COM (short username kudu)
23:31:09.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:09.138803 2582 webserver.cc:460] Webserver started at http://127.2.108.61:33735/ using document root <none> and password file <none>
23:31:09.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:09.139672 2582 fs_manager.cc:359] Metadata directory not provided
23:31:09.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:09.139840 2582 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:31:09.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:09.140202 2582 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
23:31:09.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:09.144802 2582 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-1/data/instance:
23:31:09.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6fba5a8478374b1fb2c7b5a8ff282d2b"
23:31:09.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 23:31:09 on dist-test-slave-dnlc"
23:31:09.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:09.145919 2582 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-1/wal/instance:
23:31:09.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6fba5a8478374b1fb2c7b5a8ff282d2b"
23:31:09.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 23:31:09 on dist-test-slave-dnlc"
23:31:09.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:09.153033 2582 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.001s
23:31:09.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:09.158257 2602 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
23:31:09.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:09.159580 2582 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
23:31:09.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:09.159905 2582 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-1/data,/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-1/wal
23:31:09.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6fba5a8478374b1fb2c7b5a8ff282d2b"
23:31:09.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 23:31:09 on dist-test-slave-dnlc"
23:31:09.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:09.160241 2582 fs_report.cc:383] FS layout report
23:31:09.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:31:09.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-1/wal
23:31:09.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-1/wal
23:31:09.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-1/data/data
23:31:09.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:31:09.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:31:09.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:31:09.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:31:09.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:31:09.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:31:09.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:31:09.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:31:09.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:31:09.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:31:09.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:31:09.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:09.214385 2582 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
23:31:09.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:09.220615 2582 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:31:09.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:09.221220 2582 kserver.cc:163] Server-wide thread pool size limit: 3276
23:31:09.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:09.299050 2653 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.2.108.61:33947 every 8 connection(s)
23:31:09.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:09.299044 2582 rpc_server.cc:307] RPC server started. Bound to: 127.2.108.61:33947
23:31:09.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:09.302326 2582 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-1/data/info.pb
23:31:09.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:09.308161 2654 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
23:31:09.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:09.309522 2480 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskpPnkEK/build/tsan/bin/kudu as pid 2582
23:31:09.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:09.309969 2480 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-1/wal/instance
23:31:09.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:09.320546 2654 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.108.62" port: 39471 } has no permanent_uuid. Determining permanent_uuid...
23:31:09.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.2.108.60@KRBTEST.COM; defaulting to no policy
23:31:09.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
23:31:09.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.2.108.60@KRBTEST.COM" created.
23:31:09.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 23:31:09 dist-test-slave-dnlc krb5kdc[2493](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790551869, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.108.61@KRBTEST.COM for kudu/127.2.108.62@KRBTEST.COM
23:31:09.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
23:31:09.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.108.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/krb5kdc/kudu_127.2.108.60.keytab.
23:31:09.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.108.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/krb5kdc/kudu_127.2.108.60.keytab.
23:31:09.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:09.357344 2480 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskpPnkEK/build/tsan/bin/kudu
23:31:09.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskpPnkEK/build/tsan/bin/kudu
23:31:09.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-2/wal
23:31:09.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-2/data
23:31:09.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:31:09.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:31:09.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:31:09.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:31:09.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:31:09.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:31:09.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-2/logs
23:31:09.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-2/data/info.pb
23:31:09.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:31:09.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:31:09.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:31:09.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:31:09.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:31:09.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:31:09.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:31:09.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:31:09.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
23:31:09.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:31:09.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:31:09.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:31:09.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.108.60:42421
23:31:09.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.108.60
23:31:09.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:31:09.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/krb5kdc/kudu_127.2.108.60.keytab
23:31:09.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.108.60
23:31:09.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
23:31:09.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
23:31:09.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
23:31:09.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.108.20:43467
23:31:09.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:31:09.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:31:09.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:31:09.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:31:09.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.108.62:39471,127.2.108.61:33947,127.2.108.60:42421
23:31:09.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:31:09.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44517/jwks.json
23:31:09.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
23:31:09.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:09.358274 2654 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.108.61" port: 33947 } has no permanent_uuid. Determining permanent_uuid...
23:31:09.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 23:31:09 dist-test-slave-dnlc krb5kdc[2493](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790551869, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.108.61@KRBTEST.COM for kudu/127.2.108.61@KRBTEST.COM
23:31:09.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:09.388044 2654 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.108.60" port: 42421 } has no permanent_uuid. Determining permanent_uuid...
23:31:09.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:09.390741 2605 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.108.60:42421: connect: Connection refused (error 111)
23:31:09.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:09.394531 2654 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.108.60:42421: Network error: Client connection negotiation failed: client connection to 127.2.108.60:42421: connect: Connection refused (error 111)
23:31:09.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:09.444022 2654 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.108.60" port: 42421 } attempt: 1
23:31:09.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:09.447844 2654 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.108.60:42421: Network error: Client connection negotiation failed: client connection to 127.2.108.60:42421: connect: Connection refused (error 111)
23:31:09.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:09.516193 2654 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.108.60" port: 42421 } attempt: 2
23:31:09.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:09.520843 2654 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.108.60:42421: Network error: Client connection negotiation failed: client connection to 127.2.108.60:42421: connect: Connection refused (error 111)
23:31:09.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:09.612239 2654 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.108.60" port: 42421 } attempt: 3
23:31:09.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:09.616075 2654 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.108.60:42421: Network error: Client connection negotiation failed: client connection to 127.2.108.60:42421: connect: Connection refused (error 111)
23:31:09.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:09.722417 2661 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:31:09.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:09.722935 2661 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:31:09.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:09.723511 2661 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:31:09.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:09.754242 2661 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
23:31:09.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:09.754472 2661 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
23:31:09.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:09.754724 2661 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:31:09.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:09.754910 2661 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
23:31:09.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:09.755089 2661 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
23:31:09.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:09.755285 2661 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
23:31:09.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:09.755417 2661 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44517/jwks.json
23:31:09.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:09.759541 2654 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.108.60" port: 42421 } attempt: 4
23:31:09.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:09.763129 2654 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.108.60:42421: Network error: Client connection negotiation failed: client connection to 127.2.108.60:42421: connect: Connection refused (error 111)
23:31:09.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:09.794092 2661 master_runner.cc:386] Master server non-default flags:
23:31:09.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:31:09.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.108.20:43467
23:31:09.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:31:09.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:31:09.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-2/data
23:31:09.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-2/wal
23:31:09.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
23:31:09.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:31:09.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.108.62:39471,127.2.108.61:33947,127.2.108.60:42421
23:31:09.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/krb5kdc/kudu_127.2.108.60.keytab
23:31:09.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.108.60
23:31:09.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:31:09.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:31:09.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:31:09.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.108.60:42421
23:31:09.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:31:09.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:31:09.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:31:09.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44517/jwks.json
23:31:09.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
23:31:09.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
23:31:09.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
23:31:09.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:31:09.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:31:09.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-2/data/info.pb
23:31:09.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.108.60
23:31:09.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:31:09.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:31:09.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:31:09.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:31:09.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:31:09.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:31:09.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-2/logs
23:31:09.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:31:09.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:31:09.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
23:31:09.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
23:31:09.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:31:09.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:31:09.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:31:09.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 22:42:10 UTC on ccycloud.quasar-yawxpd.root.comops.site
23:31:09.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676391
23:31:09.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:31:09.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:09.795125 2661 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:31:09.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:09.796015 2661 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:31:09.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:09.797272 2576 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.108.61" port: 33947 } attempt: 7
23:31:09.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 23:31:09 dist-test-slave-dnlc krb5kdc[2493](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790551867, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.108.62@KRBTEST.COM for kudu/127.2.108.61@KRBTEST.COM
23:31:09.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:09.818248 2672 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:31:09.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:09.824771 2576 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.108.60" port: 42421 } has no permanent_uuid. Determining permanent_uuid...
23:31:09.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:09.830039 2576 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.108.60:42421: Network error: Client connection negotiation failed: client connection to 127.2.108.60:42421: connect: Connection refused (error 111)
23:31:09.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:09.888461 2576 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.108.60" port: 42421 } attempt: 1
23:31:09.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:09.892709 2576 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.108.60:42421: Network error: Client connection negotiation failed: client connection to 127.2.108.60:42421: connect: Connection refused (error 111)
23:31:09.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:09.974077 2576 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.108.60" port: 42421 } attempt: 2
23:31:09.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:09.977563 2576 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.108.60:42421: Network error: Client connection negotiation failed: client connection to 127.2.108.60:42421: connect: Connection refused (error 111)
23:31:10.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:10.062482 2654 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.108.60" port: 42421 } attempt: 5
23:31:10.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:10.062979 2576 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.108.60" port: 42421 } attempt: 3
23:31:10.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:10.066381 2654 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.108.60:42421: Network error: Client connection negotiation failed: client connection to 127.2.108.60:42421: connect: Connection refused (error 111)
23:31:10.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:10.067979 2576 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.108.60:42421: Network error: Client connection negotiation failed: client connection to 127.2.108.60:42421: connect: Connection refused (error 111)
23:31:10.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:10.208539 2576 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.108.60" port: 42421 } attempt: 4
23:31:10.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:10.213084 2576 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.108.60:42421: Network error: Client connection negotiation failed: client connection to 127.2.108.60:42421: connect: Connection refused (error 111)
23:31:10.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:10.496495 2576 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.108.60" port: 42421 } attempt: 5
23:31:10.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:10.500726 2576 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.108.60:42421: Network error: Client connection negotiation failed: client connection to 127.2.108.60:42421: connect: Connection refused (error 111)
23:31:10.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:10.613758 2654 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.108.60" port: 42421 } attempt: 6
23:31:10.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:10.620185 2654 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.108.60:42421: Network error: Client connection negotiation failed: client connection to 127.2.108.60:42421: connect: Connection refused (error 111)
23:31:10.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:10.680495 2465 debug-util.cc:398] Leaking SignalData structure 0x7b08000b5c20 after lost signal to thread 2340
23:31:10.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:10.681188 2465 debug-util.cc:398] Leaking SignalData structure 0x7b08000b5bc0 after lost signal to thread 2468
23:31:11.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:11.053112 2576 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.108.60" port: 42421 } attempt: 6
23:31:11.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:11.063972 2576 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.108.60:42421: Network error: Client connection negotiation failed: client connection to 127.2.108.60:42421: connect: Connection refused (error 111)
23:31:11.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:09.820137 2669 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:31:11.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:09.821229 2661 server_base.cc:1034] running on GCE node
23:31:11.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:09.819051 2670 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:31:11.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:11.145231 2661 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:31:11.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:11.148908 2661 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:31:11.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:11.150520 2661 hybrid_clock.cc:648] HybridClock initialized: now 1790551871150486 us; error 42 us; skew 500 ppm
23:31:11.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:11.151671 2661 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:31:11.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 23:31:11 dist-test-slave-dnlc krb5kdc[2493](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790551871, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.108.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
23:31:11.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:11.158882 2661 init.cc:377] Logged in from keytab as kudu/127.2.108.60@KRBTEST.COM (short username kudu)
23:31:11.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:11.165514 2661 webserver.cc:460] Webserver started at http://127.2.108.60:44385/ using document root <none> and password file <none>
23:31:11.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:11.166473 2661 fs_manager.cc:359] Metadata directory not provided
23:31:11.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:11.166642 2661 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:31:11.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:11.167008 2661 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
23:31:11.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:11.171510 2661 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-2/data/instance:
23:31:11.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5739a890599e4735b0d4529f0064cdc2"
23:31:11.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 23:31:11 on dist-test-slave-dnlc"
23:31:11.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:11.172639 2661 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-2/wal/instance:
23:31:11.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5739a890599e4735b0d4529f0064cdc2"
23:31:11.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 23:31:11 on dist-test-slave-dnlc"
23:31:11.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:11.180699 2661 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.004s
23:31:11.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:11.187402 2688 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
23:31:11.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:11.188596 2661 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.002s sys 0.003s
23:31:11.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:11.188936 2661 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-2/data,/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-2/wal
23:31:11.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5739a890599e4735b0d4529f0064cdc2"
23:31:11.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 23:31:11 on dist-test-slave-dnlc"
23:31:11.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:11.189280 2661 fs_report.cc:383] FS layout report
23:31:11.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:31:11.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-2/wal
23:31:11.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-2/wal
23:31:11.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-2/data/data
23:31:11.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:31:11.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:31:11.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:31:11.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:31:11.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:31:11.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:31:11.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:31:11.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:31:11.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:31:11.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:31:11.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:31:11.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:11.233544 2661 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
23:31:11.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:11.239349 2661 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:31:11.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:11.239897 2661 kserver.cc:163] Server-wide thread pool size limit: 3276
23:31:11.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:11.336794 2661 rpc_server.cc:307] RPC server started. Bound to: 127.2.108.60:42421
23:31:11.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:11.336871 2739 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.2.108.60:42421 every 8 connection(s)
23:31:11.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:11.340716 2661 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-2/data/info.pb
23:31:11.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:11.344775 2480 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskpPnkEK/build/tsan/bin/kudu as pid 2661
23:31:11.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:11.345266 2480 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-2/wal/instance
23:31:11.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:11.357929 2740 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
23:31:11.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:11.380019 2740 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.108.62" port: 39471 } has no permanent_uuid. Determining permanent_uuid...
23:31:11.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 23:31:11 dist-test-slave-dnlc krb5kdc[2493](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790551865, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.2.108.60@KRBTEST.COM
23:31:11.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 23:31:11 dist-test-slave-dnlc krb5kdc[2493](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790551871, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.108.60@KRBTEST.COM for kudu/127.2.108.62@KRBTEST.COM
23:31:11.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:11.421236 2740 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.108.61" port: 33947 } has no permanent_uuid. Determining permanent_uuid...
23:31:11.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 23:31:11 dist-test-slave-dnlc krb5kdc[2493](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790551865, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.2.108.61@KRBTEST.COM
23:31:11.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 23:31:11 dist-test-slave-dnlc krb5kdc[2493](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790551871, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.108.60@KRBTEST.COM for kudu/127.2.108.61@KRBTEST.COM
23:31:11.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:11.451419 2740 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.108.60" port: 42421 } has no permanent_uuid. Determining permanent_uuid...
23:31:11.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 23:31:11 dist-test-slave-dnlc krb5kdc[2493](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790551871, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.108.60@KRBTEST.COM for kudu/127.2.108.60@KRBTEST.COM
23:31:11.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 23:31:11 dist-test-slave-dnlc krb5kdc[2493](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790551865, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.2.108.62@KRBTEST.COM
23:31:11.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:11.485085 2740 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2: Bootstrap starting.
23:31:11.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:11.492894 2740 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2: Neither blocks nor log segments found. Creating new log.
23:31:11.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:11.495265 2740 log.cc:824] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2: Log is configured to *not* fsync() on all Append() calls
23:31:11.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:11.499521 2480 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
23:31:11.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:11.501348 2740 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2: No bootstrap required, opened a new log
23:31:11.506 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
23:31:11.508 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
23:31:11.519 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
23:31:11.520 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
23:31:11.523 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.2.108.62"
port: 39471
}
bound_http_address {
host: "127.2.108.62"
port: 34899
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.2.108.61"
port: 33947
}
bound_http_address {
host: "127.2.108.61"
port: 33735
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.2.108.60"
port: 42421
}
bound_http_address {
host: "127.2.108.60"
port: 44385
}
}
}
23:31:11.524 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
23:31:11.525 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
23:31:11.525 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
23:31:11.527 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
23:31:11.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:11.534963 2740 raft_consensus.cc:348] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f6cda9ea914a448ebe7d4c25dec6364f" member_type: VOTER last_known_addr { host: "127.2.108.62" port: 39471 } } peers { permanent_uuid: "6fba5a8478374b1fb2c7b5a8ff282d2b" member_type: VOTER last_known_addr { host: "127.2.108.61" port: 33947 } } peers { permanent_uuid: "5739a890599e4735b0d4529f0064cdc2" member_type: VOTER last_known_addr { host: "127.2.108.60" port: 42421 } }
23:31:11.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:11.535913 2740 raft_consensus.cc:374] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
23:31:11.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:11.536218 2740 raft_consensus.cc:729] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5739a890599e4735b0d4529f0064cdc2, State: Initialized, Role: FOLLOWER
23:31:11.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:11.537297 2740 consensus_queue.cc:260] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f6cda9ea914a448ebe7d4c25dec6364f" member_type: VOTER last_known_addr { host: "127.2.108.62" port: 39471 } } peers { permanent_uuid: "6fba5a8478374b1fb2c7b5a8ff282d2b" member_type: VOTER last_known_addr { host: "127.2.108.61" port: 33947 } } peers { permanent_uuid: "5739a890599e4735b0d4529f0064cdc2" member_type: VOTER last_known_addr { host: "127.2.108.60" port: 42421 } }
23:31:11.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:11.540823 2750 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f6cda9ea914a448ebe7d4c25dec6364f" member_type: VOTER last_known_addr { host: "127.2.108.62" port: 39471 } } peers { permanent_uuid: "6fba5a8478374b1fb2c7b5a8ff282d2b" member_type: VOTER last_known_addr { host: "127.2.108.61" port: 33947 } } peers { permanent_uuid: "5739a890599e4735b0d4529f0064cdc2" member_type: VOTER last_known_addr { host: "127.2.108.60" port: 42421 } } }
23:31:11.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:11.541862 2750 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2 [sys.catalog]: This master's current role is: FOLLOWER
23:31:11.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:11.543043 2740 sys_catalog.cc:564] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2 [sys.catalog]: configured and running, proceeding with master startup.
23:31:11.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 23:31:11 dist-test-slave-dnlc krb5kdc[2493](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790551871, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
23:31:11.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
23:31:11.563 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
23:31:11.565 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/krb5kdc/krb5cc
23:31:11.574 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
23:31:11.576 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/krb5kdc/krb5cc
23:31:11.577 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
23:31:11.582 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
23:31:11.584 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
23:31:11.585 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
23:31:11.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:11.606011 2762 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:31:11.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:11.607674 2762 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:31:11.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:11.575268 2661 master_runner.cc:186] Error getting master registration for 127.2.108.62:39471: OK, instance_id { permanent_uuid: "f6cda9ea914a448ebe7d4c25dec6364f" instance_seqno: 1790551867414222 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
23:31:11.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:11.611037 2661 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
23:31:11.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 23:31:11 dist-test-slave-dnlc krb5kdc[2493](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790551871, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
23:31:11.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
23:31:11.652 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
23:31:11.652 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/krb5kdc/krb5cc
23:31:11.654 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
23:31:11.654 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.2.108.60:42421
23:31:11.656 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
23:31:11.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:11.657219 2480 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskpPnkEK/build/tsan/bin/kudu with pid 2661
23:31:11.676 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
23:31:11.676 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.2.108.62:39471
23:31:11.677 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
23:31:11.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:11.677287 2480 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskpPnkEK/build/tsan/bin/kudu with pid 2506
23:31:11.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:11.680598 2654 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.108.60" port: 42421 } attempt: 7
23:31:11.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:11.689399 2654 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.108.60:42421: Network error: Client connection negotiation failed: client connection to 127.2.108.60:42421: connect: Connection refused (error 111)
23:31:11.706 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
23:31:11.706 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.2.108.61:33947
23:31:11.707 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
23:31:11.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:11.707311 2480 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskpPnkEK/build/tsan/bin/kudu with pid 2582
23:31:11.734 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
23:31:11.734 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.2.108.60:42421
23:31:11.737 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
23:31:11.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:11.738852 2480 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskpPnkEK/build/tsan/bin/kudu
23:31:11.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskpPnkEK/build/tsan/bin/kudu
23:31:11.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-2/wal
23:31:11.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-2/data
23:31:11.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:31:11.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:31:11.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:31:11.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:31:11.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:31:11.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:31:11.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-2/logs
23:31:11.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-2/data/info.pb
23:31:11.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:31:11.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:31:11.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:31:11.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:31:11.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:31:11.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:31:11.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:31:11.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:31:11.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
23:31:11.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:31:11.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:31:11.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:31:11.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.108.60:42421
23:31:11.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.108.60
23:31:11.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44385
23:31:11.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/krb5kdc/kudu_127.2.108.60.keytab
23:31:11.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.108.60
23:31:11.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
23:31:11.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
23:31:11.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
23:31:11.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.108.20:43467
23:31:11.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:31:11.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:31:11.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:31:11.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:31:11.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.108.62:39471,127.2.108.61:33947,127.2.108.60:42421
23:31:11.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:31:11.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44517/jwks.json
23:31:11.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
23:31:12.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:12.104115 2768 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:31:12.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:12.104671 2768 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:31:12.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:12.105423 2768 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:31:12.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:12.137553 2768 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
23:31:12.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:12.137944 2768 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
23:31:12.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:12.138244 2768 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:31:12.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:12.138447 2768 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
23:31:12.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:12.138656 2768 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
23:31:12.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:12.138855 2768 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
23:31:12.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:12.138996 2768 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44517/jwks.json
23:31:12.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:12.175117 2768 master_runner.cc:386] Master server non-default flags:
23:31:12.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:31:12.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.108.20:43467
23:31:12.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:31:12.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:31:12.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-2/data
23:31:12.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-2/wal
23:31:12.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
23:31:12.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:31:12.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.108.62:39471,127.2.108.61:33947,127.2.108.60:42421
23:31:12.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/krb5kdc/kudu_127.2.108.60.keytab
23:31:12.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.108.60
23:31:12.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:31:12.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:31:12.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:31:12.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.108.60:42421
23:31:12.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:31:12.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:31:12.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:31:12.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44517/jwks.json
23:31:12.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
23:31:12.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
23:31:12.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
23:31:12.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:31:12.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:31:12.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-2/data/info.pb
23:31:12.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.108.60
23:31:12.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44385
23:31:12.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:31:12.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:31:12.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:31:12.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:31:12.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:31:12.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-2/logs
23:31:12.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:31:12.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:31:12.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
23:31:12.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
23:31:12.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:31:12.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:31:12.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:31:12.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 22:42:10 UTC on ccycloud.quasar-yawxpd.root.comops.site
23:31:12.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676391
23:31:12.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:31:12.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:12.176186 2768 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:31:12.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:12.177100 2768 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:31:12.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:12.196898 2776 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:31:13.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:12.201761 2775 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:31:13.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:12.196956 2778 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:31:13.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:13.557440 2777 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1358 milliseconds
23:31:13.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:13.557559 2768 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
23:31:13.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:13.558945 2768 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:31:13.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:13.562058 2768 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:31:13.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:13.563454 2768 hybrid_clock.cc:648] HybridClock initialized: now 1790551873563421 us; error 57 us; skew 500 ppm
23:31:13.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:13.564193 2768 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:31:13.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 23:31:13 dist-test-slave-dnlc krb5kdc[2493](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790551873, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.108.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
23:31:13.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:13.574505 2768 init.cc:377] Logged in from keytab as kudu/127.2.108.60@KRBTEST.COM (short username kudu)
23:31:13.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:13.579485 2768 webserver.cc:460] Webserver started at http://127.2.108.60:44385/ using document root <none> and password file <none>
23:31:13.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:13.580521 2768 fs_manager.cc:359] Metadata directory not provided
23:31:13.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:13.580765 2768 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:31:13.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:13.588794 2768 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.006s sys 0.000s
23:31:13.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:13.593134 2789 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
23:31:13.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:13.594252 2768 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
23:31:13.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:13.594538 2768 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-2/data,/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-2/wal
23:31:13.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5739a890599e4735b0d4529f0064cdc2"
23:31:13.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 23:31:11 on dist-test-slave-dnlc"
23:31:13.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:13.597154 2768 fs_report.cc:383] FS layout report
23:31:13.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:31:13.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-2/wal
23:31:13.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-2/wal
23:31:13.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-2/data/data
23:31:13.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:31:13.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:31:13.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:31:13.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:31:13.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:31:13.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:31:13.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:31:13.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:31:13.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:31:13.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:31:13.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:31:13.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:13.632236 2768 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
23:31:13.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:13.639247 2768 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:31:13.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:13.639746 2768 kserver.cc:163] Server-wide thread pool size limit: 3276
23:31:13.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:13.714888 2768 rpc_server.cc:307] RPC server started. Bound to: 127.2.108.60:42421
23:31:13.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:13.715036 2840 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.2.108.60:42421 every 8 connection(s)
23:31:13.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:13.717831 2768 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-2/data/info.pb
23:31:13.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:13.722468 2480 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskpPnkEK/build/tsan/bin/kudu as pid 2768
23:31:13.723 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
23:31:13.723 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.2.108.62:39471
23:31:13.723 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
23:31:13.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:13.724967 2480 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskpPnkEK/build/tsan/bin/kudu
23:31:13.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskpPnkEK/build/tsan/bin/kudu
23:31:13.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-0/wal
23:31:13.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-0/data
23:31:13.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:31:13.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:31:13.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:31:13.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:31:13.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:31:13.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:31:13.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-0/logs
23:31:13.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-0/data/info.pb
23:31:13.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:31:13.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:31:13.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:31:13.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:31:13.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:31:13.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:31:13.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:31:13.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:31:13.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
23:31:13.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:31:13.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:31:13.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:31:13.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.108.62:39471
23:31:13.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.108.62
23:31:13.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34899
23:31:13.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/krb5kdc/kudu_127.2.108.62.keytab
23:31:13.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.108.62
23:31:13.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
23:31:13.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
23:31:13.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
23:31:13.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.108.20:43467
23:31:13.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:31:13.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:31:13.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:31:13.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:31:13.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.108.62:39471,127.2.108.61:33947,127.2.108.60:42421
23:31:13.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:31:13.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44517/jwks.json
23:31:13.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
23:31:13.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:13.729978 2841 sys_catalog.cc:263] Verifying existing consensus state
23:31:13.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:13.739382 2841 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2: Bootstrap starting.
23:31:13.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:13.775373 2841 log.cc:824] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2: Log is configured to *not* fsync() on all Append() calls
23:31:13.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:13.780162 2841 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
23:31:13.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:13.780901 2841 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2: Bootstrap complete.
23:31:13.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:13.804412 2841 raft_consensus.cc:348] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f6cda9ea914a448ebe7d4c25dec6364f" member_type: VOTER last_known_addr { host: "127.2.108.62" port: 39471 } } peers { permanent_uuid: "6fba5a8478374b1fb2c7b5a8ff282d2b" member_type: VOTER last_known_addr { host: "127.2.108.61" port: 33947 } } peers { permanent_uuid: "5739a890599e4735b0d4529f0064cdc2" member_type: VOTER last_known_addr { host: "127.2.108.60" port: 42421 } }
23:31:13.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:13.807070 2841 raft_consensus.cc:374] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
23:31:13.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:13.807370 2841 raft_consensus.cc:729] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5739a890599e4735b0d4529f0064cdc2, State: Initialized, Role: FOLLOWER
23:31:13.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:13.808241 2841 consensus_queue.cc:260] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f6cda9ea914a448ebe7d4c25dec6364f" member_type: VOTER last_known_addr { host: "127.2.108.62" port: 39471 } } peers { permanent_uuid: "6fba5a8478374b1fb2c7b5a8ff282d2b" member_type: VOTER last_known_addr { host: "127.2.108.61" port: 33947 } } peers { permanent_uuid: "5739a890599e4735b0d4529f0064cdc2" member_type: VOTER last_known_addr { host: "127.2.108.60" port: 42421 } }
23:31:13.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:13.811328 2845 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f6cda9ea914a448ebe7d4c25dec6364f" member_type: VOTER last_known_addr { host: "127.2.108.62" port: 39471 } } peers { permanent_uuid: "6fba5a8478374b1fb2c7b5a8ff282d2b" member_type: VOTER last_known_addr { host: "127.2.108.61" port: 33947 } } peers { permanent_uuid: "5739a890599e4735b0d4529f0064cdc2" member_type: VOTER last_known_addr { host: "127.2.108.60" port: 42421 } } }
23:31:13.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:13.812179 2845 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2 [sys.catalog]: This master's current role is: FOLLOWER
23:31:13.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:13.813083 2841 sys_catalog.cc:564] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2 [sys.catalog]: configured and running, proceeding with master startup.
23:31:13.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:13.850929 2856 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
23:31:13.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:13.856998 2768 master_runner.cc:186] Error getting master registration for 127.2.108.62:39471: Network error: Client connection negotiation failed: client connection to 127.2.108.62:39471: connect: Connection refused (error 111),
23:31:13.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:13.878938 2768 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
23:31:13.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:13.895648 2795 tablet.cc:2369] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2: Can't schedule compaction. Clean time has not been advanced past its initial value.
23:31:13.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:13.964955 2845 raft_consensus.cc:482] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
23:31:13.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:13.965423 2845 raft_consensus.cc:504] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f6cda9ea914a448ebe7d4c25dec6364f" member_type: VOTER last_known_addr { host: "127.2.108.62" port: 39471 } } peers { permanent_uuid: "6fba5a8478374b1fb2c7b5a8ff282d2b" member_type: VOTER last_known_addr { host: "127.2.108.61" port: 33947 } } peers { permanent_uuid: "5739a890599e4735b0d4529f0064cdc2" member_type: VOTER last_known_addr { host: "127.2.108.60" port: 42421 } }
23:31:13.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:13.968005 2845 leader_election.cc:288] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f6cda9ea914a448ebe7d4c25dec6364f (127.2.108.62:39471), 6fba5a8478374b1fb2c7b5a8ff282d2b (127.2.108.61:33947)
23:31:13.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:13.969587 2793 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.108.62:39471: connect: Connection refused (error 111)
23:31:13.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:13.974236 2793 leader_election.cc:334] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer f6cda9ea914a448ebe7d4c25dec6364f (127.2.108.62:39471): Network error: Client connection negotiation failed: client connection to 127.2.108.62:39471: connect: Connection refused (error 111)
23:31:13.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:13.974812 2791 leader_election.cc:334] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 6fba5a8478374b1fb2c7b5a8ff282d2b (127.2.108.61:33947): Network error: Client connection negotiation failed: client connection to 127.2.108.61:33947: connect: Connection refused (error 111)
23:31:13.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:13.975167 2791 leader_election.cc:302] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5739a890599e4735b0d4529f0064cdc2; no voters: 6fba5a8478374b1fb2c7b5a8ff282d2b, f6cda9ea914a448ebe7d4c25dec6364f
23:31:13.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:13.975870 2845 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
23:31:14.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:14.063563 2843 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:31:14.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:14.064095 2843 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:31:14.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:14.064750 2843 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:31:14.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:14.095994 2843 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
23:31:14.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:14.096227 2843 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
23:31:14.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:14.096472 2843 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:31:14.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:14.096681 2843 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
23:31:14.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:14.096876 2843 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
23:31:14.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:14.097056 2843 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
23:31:14.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:14.097194 2843 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44517/jwks.json
23:31:14.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:14.132388 2843 master_runner.cc:386] Master server non-default flags:
23:31:14.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:31:14.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.108.20:43467
23:31:14.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:31:14.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:31:14.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-0/data
23:31:14.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-0/wal
23:31:14.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
23:31:14.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:31:14.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.108.62:39471,127.2.108.61:33947,127.2.108.60:42421
23:31:14.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/krb5kdc/kudu_127.2.108.62.keytab
23:31:14.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.108.62
23:31:14.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:31:14.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:31:14.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:31:14.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.108.62:39471
23:31:14.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:31:14.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:31:14.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:31:14.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44517/jwks.json
23:31:14.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
23:31:14.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
23:31:14.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
23:31:14.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:31:14.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:31:14.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-0/data/info.pb
23:31:14.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.108.62
23:31:14.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34899
23:31:14.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:31:14.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:31:14.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:31:14.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:31:14.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:31:14.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-0/logs
23:31:14.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:31:14.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:31:14.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
23:31:14.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
23:31:14.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:31:14.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:31:14.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:31:14.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 22:42:10 UTC on ccycloud.quasar-yawxpd.root.comops.site
23:31:14.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676391
23:31:14.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:31:14.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:14.133450 2843 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:31:14.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:14.134730 2843 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:31:14.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:14.152892 2865 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:31:14.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:14.859076 2856 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:31:14.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:14.859408 2856 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:31:15.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:14.153057 2866 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:31:15.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:14.154685 2843 server_base.cc:1034] running on GCE node
23:31:15.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:14.153532 2868 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:31:15.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:15.394083 2843 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:31:15.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:15.397234 2843 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:31:15.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:15.398825 2843 hybrid_clock.cc:648] HybridClock initialized: now 1790551875398787 us; error 40 us; skew 500 ppm
23:31:15.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:15.399940 2843 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:31:15.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 23:31:15 dist-test-slave-dnlc krb5kdc[2493](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790551875, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.108.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
23:31:15.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:15.406929 2843 init.cc:377] Logged in from keytab as kudu/127.2.108.62@KRBTEST.COM (short username kudu)
23:31:15.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:15.413082 2843 webserver.cc:460] Webserver started at http://127.2.108.62:34899/ using document root <none> and password file <none>
23:31:15.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:15.414072 2843 fs_manager.cc:359] Metadata directory not provided
23:31:15.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:15.414309 2843 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:31:15.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:15.422803 2843 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.007s sys 0.001s
23:31:15.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:15.427735 2879 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
23:31:15.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:15.428785 2843 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.001s sys 0.004s
23:31:15.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:15.429044 2843 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-0/data,/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-0/wal
23:31:15.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f6cda9ea914a448ebe7d4c25dec6364f"
23:31:15.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 23:31:07 on dist-test-slave-dnlc"
23:31:15.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:15.431264 2843 fs_report.cc:383] FS layout report
23:31:15.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:31:15.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-0/wal
23:31:15.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-0/wal
23:31:15.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-0/data/data
23:31:15.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:31:15.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:31:15.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:31:15.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:31:15.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:31:15.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:31:15.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:31:15.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:31:15.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:31:15.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:31:15.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:31:15.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:15.498888 2843 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
23:31:15.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:15.505349 2843 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:31:15.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:15.505899 2843 kserver.cc:163] Server-wide thread pool size limit: 3276
23:31:15.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:15.587265 2843 rpc_server.cc:307] RPC server started. Bound to: 127.2.108.62:39471
23:31:15.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:15.587715 2930 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.2.108.62:39471 every 8 connection(s)
23:31:15.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:15.590184 2843 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-0/data/info.pb
23:31:15.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:15.591941 2480 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskpPnkEK/build/tsan/bin/kudu as pid 2843
23:31:15.594 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
23:31:15.594 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.2.108.61:33947
23:31:15.594 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
23:31:15.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:15.597889 2480 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskpPnkEK/build/tsan/bin/kudu
23:31:15.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskpPnkEK/build/tsan/bin/kudu
23:31:15.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-1/wal
23:31:15.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-1/data
23:31:15.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:31:15.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:31:15.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:31:15.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:31:15.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:31:15.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:31:15.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-1/logs
23:31:15.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-1/data/info.pb
23:31:15.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:31:15.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:31:15.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:31:15.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:31:15.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:31:15.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:31:15.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:31:15.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:31:15.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
23:31:15.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:31:15.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:31:15.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:31:15.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.108.61:33947
23:31:15.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.108.61
23:31:15.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33735
23:31:15.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/krb5kdc/kudu_127.2.108.61.keytab
23:31:15.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.108.61
23:31:15.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
23:31:15.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
23:31:15.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
23:31:15.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.108.20:43467
23:31:15.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:31:15.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:31:15.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:31:15.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:31:15.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.108.62:39471,127.2.108.61:33947,127.2.108.60:42421
23:31:15.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:31:15.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44517/jwks.json
23:31:15.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
23:31:15.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:15.604346 2931 sys_catalog.cc:263] Verifying existing consensus state
23:31:15.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260927 23:31:15.605989 2931 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
23:31:15.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:15.606971 2843 master.cc:537] Master@127.2.108.62:39471 shutting down...
23:31:15.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:15.698513 2843 master.cc:552] Master@127.2.108.62:39471 shutdown complete.
23:31:15.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
23:31:15.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:15.863473 2856 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:31:15.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:15.863790 2856 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:31:15.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:15.962248 2933 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:31:15.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:15.962764 2933 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:31:15.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:15.963332 2933 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:31:15.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:15.993993 2933 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
23:31:15.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:15.994197 2933 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
23:31:15.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:15.994467 2933 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:31:15.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:15.994819 2933 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
23:31:15.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:15.995174 2933 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
23:31:15.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:15.995572 2933 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
23:31:15.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:15.995859 2933 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44517/jwks.json
23:31:16.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:16.056910 2933 master_runner.cc:386] Master server non-default flags:
23:31:16.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:31:16.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.108.20:43467
23:31:16.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:31:16.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:31:16.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-1/data
23:31:16.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-1/wal
23:31:16.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
23:31:16.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:31:16.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.108.62:39471,127.2.108.61:33947,127.2.108.60:42421
23:31:16.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/krb5kdc/kudu_127.2.108.61.keytab
23:31:16.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.108.61
23:31:16.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:31:16.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:31:16.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:31:16.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.108.61:33947
23:31:16.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:31:16.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:31:16.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:31:16.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44517/jwks.json
23:31:16.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
23:31:16.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
23:31:16.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
23:31:16.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:31:16.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:31:16.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-1/data/info.pb
23:31:16.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.108.61
23:31:16.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33735
23:31:16.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:31:16.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:31:16.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:31:16.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:31:16.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:31:16.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-1/logs
23:31:16.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:31:16.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:31:16.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
23:31:16.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
23:31:16.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:31:16.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
23:31:16.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:31:16.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 22:42:10 UTC on ccycloud.quasar-yawxpd.root.comops.site
23:31:16.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676391
23:31:16.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:31:16.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:16.058372 2933 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:31:16.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:16.059530 2933 file_cache.cc:493] Constructed file cache file cache with capacity 419430
23:31:16.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:16.088081 2939 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:31:16.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:16.216805 2945 raft_consensus.cc:482] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
23:31:16.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:16.219789 2945 raft_consensus.cc:504] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f6cda9ea914a448ebe7d4c25dec6364f" member_type: VOTER last_known_addr { host: "127.2.108.62" port: 39471 } } peers { permanent_uuid: "6fba5a8478374b1fb2c7b5a8ff282d2b" member_type: VOTER last_known_addr { host: "127.2.108.61" port: 33947 } } peers { permanent_uuid: "5739a890599e4735b0d4529f0064cdc2" member_type: VOTER last_known_addr { host: "127.2.108.60" port: 42421 } }
23:31:16.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:16.245441 2945 leader_election.cc:288] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f6cda9ea914a448ebe7d4c25dec6364f (127.2.108.62:39471), 6fba5a8478374b1fb2c7b5a8ff282d2b (127.2.108.61:33947)
23:31:16.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:16.263518 2793 leader_election.cc:334] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer f6cda9ea914a448ebe7d4c25dec6364f (127.2.108.62:39471): Network error: Client connection negotiation failed: client connection to 127.2.108.62:39471: connect: Connection refused (error 111)
23:31:16.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:16.265959 2791 leader_election.cc:334] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 6fba5a8478374b1fb2c7b5a8ff282d2b (127.2.108.61:33947): Network error: Client connection negotiation failed: client connection to 127.2.108.61:33947: connect: Connection refused (error 111)
23:31:16.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:16.266350 2791 leader_election.cc:302] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5739a890599e4735b0d4529f0064cdc2; no voters: 6fba5a8478374b1fb2c7b5a8ff282d2b, f6cda9ea914a448ebe7d4c25dec6364f
23:31:16.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:16.267019 2945 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
23:31:16.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:16.869187 2856 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:31:16.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:16.870041 2856 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:31:17.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:17.105055 2837 debug-util.cc:398] Leaking SignalData structure 0x7b0800073780 after lost signal to thread 2770
23:31:17.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:17.105826 2837 debug-util.cc:398] Leaking SignalData structure 0x7b08000b1d20 after lost signal to thread 2840
23:31:17.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:17.478935 2938 debug-util.cc:398] Leaking SignalData structure 0x7b0800014920 after lost signal to thread 2933
23:31:17.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:17.762609 2933 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 1.672s user 0.518s sys 0.789s
23:31:17.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:16.090458 2940 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:31:17.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:17.763028 2933 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 1.673s user 0.518s sys 0.789s
23:31:17.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:17.765180 2942 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:31:17.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:17.768903 2941 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
23:31:17.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:17.768936 2933 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
23:31:17.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:17.770402 2933 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:31:17.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:17.773372 2933 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:31:17.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:17.774830 2933 hybrid_clock.cc:648] HybridClock initialized: now 1790551877774797 us; error 36 us; skew 500 ppm
23:31:17.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:17.775869 2933 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:31:17.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 23:31:17 dist-test-slave-dnlc krb5kdc[2493](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790551877, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.108.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
23:31:17.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:17.783131 2933 init.cc:377] Logged in from keytab as kudu/127.2.108.61@KRBTEST.COM (short username kudu)
23:31:17.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:17.788391 2933 webserver.cc:460] Webserver started at http://127.2.108.61:33735/ using document root <none> and password file <none>
23:31:17.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:17.789217 2933 fs_manager.cc:359] Metadata directory not provided
23:31:17.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:17.789423 2933 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:31:17.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:17.797669 2933 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.005s sys 0.001s
23:31:17.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:17.801826 2955 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
23:31:17.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:17.802951 2933 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
23:31:17.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:17.803303 2933 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-1/data,/tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-1/wal
23:31:17.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6fba5a8478374b1fb2c7b5a8ff282d2b"
23:31:17.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 23:31:09 on dist-test-slave-dnlc"
23:31:17.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:17.805696 2933 fs_report.cc:383] FS layout report
23:31:17.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:31:17.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-1/wal
23:31:17.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-1/wal
23:31:17.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-1/data/data
23:31:17.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:31:17.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:31:17.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:31:17.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:31:17.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:31:17.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:31:17.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:31:17.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:31:17.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:31:17.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:31:17.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:31:17.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:17.868212 2933 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
23:31:17.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:17.873656 2856 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:31:17.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:17.873929 2856 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:31:17.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:17.876294 2933 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:31:17.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:17.876893 2933 kserver.cc:163] Server-wide thread pool size limit: 3276
23:31:17.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:17.970737 2933 rpc_server.cc:307] RPC server started. Bound to: 127.2.108.61:33947
23:31:17.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:17.970901 3006 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.2.108.61:33947 every 8 connection(s)
23:31:17.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:17.973531 2933 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-1/data/info.pb
23:31:17.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:17.983367 2480 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskpPnkEK/build/tsan/bin/kudu as pid 2933
23:31:17.984 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
23:31:17.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:17.988134 3007 sys_catalog.cc:263] Verifying existing consensus state
23:31:17.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260927 23:31:17.989138 3007 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
23:31:17.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:17.989856 2933 master.cc:537] Master@127.2.108.61:33947 shutting down...
23:31:18.043 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.2.108.60:42421(127.2.108.60:42421)] connecting to peer
23:31:18.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:18.095127 2933 master.cc:552] Master@127.2.108.61:33947 shutdown complete.
23:31:18.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskpPnkEK/test-tmp/mini-kudu-cluster2945913773980283568/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
23:31:18.130 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.2.108.62:39471(127.2.108.62:39471)] connecting to peer
23:31:18.134 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.2.108.61:33947(127.2.108.61:33947)] connecting to peer
23:31:18.173 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471
23:31:18.174 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.108.62:39471(127.2.108.62:39471)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471
23:31:18.174 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.2.108.60:42421(127.2.108.60:42421)] Successfully connected to peer
23:31:18.176 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.108.62:39471: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471
23:31:18.235 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
23:31:18.235 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.108.61:33947(127.2.108.61:33947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
23:31:18.237 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.108.61:33947: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
23:31:18.254 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
23:31:18.264 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.2.108.60
23:31:18.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 23:31:18 dist-test-slave-dnlc krb5kdc[2493](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790551871, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.2.108.60@KRBTEST.COM
23:31:18.535 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x80b33d83, L:/127.0.0.1:55390 - R:/127.2.108.60:42421] using SASL/GSSAPI
23:31:18.550 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.2.108.60:42421,127.2.108.62:39471,127.2.108.61:33947; will retry
23:31:18.571 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.108.62:39471(127.2.108.62:39471)] connecting to peer
23:31:18.572 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.108.61:33947(127.2.108.61:33947)] connecting to peer
23:31:18.576 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471
23:31:18.576 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.108.62:39471(127.2.108.62:39471)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471
23:31:18.577 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.108.62:39471: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471
23:31:18.580 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
23:31:18.580 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.108.61:33947(127.2.108.61:33947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
23:31:18.581 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.108.61:33947: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
23:31:18.581 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.2.108.60:42421,127.2.108.62:39471,127.2.108.61:33947; will retry
23:31:18.589 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.108.62:39471(127.2.108.62:39471)] connecting to peer
23:31:18.594 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.108.61:33947(127.2.108.61:33947)] connecting to peer
23:31:18.594 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471
23:31:18.594 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.108.62:39471(127.2.108.62:39471)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471
23:31:18.596 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.108.62:39471: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471
23:31:18.597 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
23:31:18.597 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.108.61:33947(127.2.108.61:33947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
23:31:18.598 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.108.61:33947: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
23:31:18.598 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.2.108.60:42421,127.2.108.62:39471,127.2.108.61:33947; will retry
23:31:18.609 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.108.62:39471(127.2.108.62:39471)] connecting to peer
23:31:18.612 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471
23:31:18.612 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.108.61:33947(127.2.108.61:33947)] connecting to peer
23:31:18.612 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.108.62:39471(127.2.108.62:39471)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471
23:31:18.613 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.108.62:39471: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471
23:31:18.617 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
23:31:18.617 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.108.61:33947(127.2.108.61:33947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
23:31:18.618 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.108.61:33947: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
23:31:18.620 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.2.108.60:42421,127.2.108.62:39471,127.2.108.61:33947; will retry
23:31:18.649 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.108.62:39471(127.2.108.62:39471)] connecting to peer
23:31:18.655 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.108.61:33947(127.2.108.61:33947)] connecting to peer
23:31:18.656 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471
23:31:18.656 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.108.62:39471(127.2.108.62:39471)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471
23:31:18.656 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.108.62:39471: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471
23:31:18.657 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
23:31:18.658 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.108.61:33947(127.2.108.61:33947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
23:31:18.659 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.108.61:33947: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
23:31:18.659 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.2.108.60:42421,127.2.108.62:39471,127.2.108.61:33947; will retry
23:31:18.689 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.108.62:39471(127.2.108.62:39471)] connecting to peer
23:31:18.691 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.108.61:33947(127.2.108.61:33947)] connecting to peer
23:31:18.692 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471
23:31:18.692 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.108.62:39471(127.2.108.62:39471)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471
23:31:18.694 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.108.62:39471: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471
23:31:18.705 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
23:31:18.705 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.108.61:33947(127.2.108.61:33947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
23:31:18.706 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.108.61:33947: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
23:31:18.706 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.2.108.60:42421,127.2.108.62:39471,127.2.108.61:33947; will retry
23:31:18.729 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.108.62:39471(127.2.108.62:39471)] connecting to peer
23:31:18.731 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.108.61:33947(127.2.108.61:33947)] connecting to peer
23:31:18.732 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471
23:31:18.732 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.108.62:39471(127.2.108.62:39471)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471
23:31:18.733 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.108.62:39471: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471
23:31:18.735 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
23:31:18.735 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.108.61:33947(127.2.108.61:33947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
23:31:18.736 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.108.61:33947: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
23:31:18.736 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.2.108.60:42421,127.2.108.62:39471,127.2.108.61:33947; will retry
23:31:18.850 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.108.62:39471(127.2.108.62:39471)] connecting to peer
23:31:18.853 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.108.61:33947(127.2.108.61:33947)] connecting to peer
23:31:18.856 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471
23:31:18.856 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.108.62:39471(127.2.108.62:39471)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471
23:31:18.857 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
23:31:18.857 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.108.62:39471: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471
23:31:18.857 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.108.61:33947(127.2.108.61:33947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
23:31:18.858 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.108.61:33947: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
23:31:18.858 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.2.108.60:42421,127.2.108.62:39471,127.2.108.61:33947; will retry
23:31:18.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:18.876643 2856 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:31:18.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:18.876902 2856 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:31:18.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:18.909960 2768 master_runner.cc:186] Error getting master registration for 127.2.108.62:39471: Network error: Client connection negotiation failed: client connection to 127.2.108.62:39471: connect: Connection refused (error 111),
23:31:18.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:18.910442 2768 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
23:31:18.970 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.108.62:39471(127.2.108.62:39471)] connecting to peer
23:31:18.970 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.108.61:33947(127.2.108.61:33947)] connecting to peer
23:31:18.972 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471
23:31:18.973 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.108.62:39471(127.2.108.62:39471)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471
23:31:18.974 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.108.62:39471: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471
23:31:18.975 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
23:31:18.975 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.108.61:33947(127.2.108.61:33947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
23:31:18.975 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.108.61:33947: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
23:31:18.976 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.2.108.60:42421,127.2.108.62:39471,127.2.108.61:33947; will retry
23:31:19.110 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.108.62:39471(127.2.108.62:39471)] connecting to peer
23:31:19.112 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471
23:31:19.112 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.108.62:39471(127.2.108.62:39471)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471
23:31:19.113 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.108.61:33947(127.2.108.61:33947)] connecting to peer
23:31:19.114 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.108.62:39471: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471
23:31:19.116 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
23:31:19.125 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.108.61:33947(127.2.108.61:33947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
23:31:19.126 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.108.61:33947: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
23:31:19.126 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.2.108.60:42421,127.2.108.62:39471,127.2.108.61:33947; will retry
23:31:19.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:19.638141 3014 raft_consensus.cc:482] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
23:31:19.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:19.638527 3014 raft_consensus.cc:504] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f6cda9ea914a448ebe7d4c25dec6364f" member_type: VOTER last_known_addr { host: "127.2.108.62" port: 39471 } } peers { permanent_uuid: "6fba5a8478374b1fb2c7b5a8ff282d2b" member_type: VOTER last_known_addr { host: "127.2.108.61" port: 33947 } } peers { permanent_uuid: "5739a890599e4735b0d4529f0064cdc2" member_type: VOTER last_known_addr { host: "127.2.108.60" port: 42421 } }
23:31:19.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:19.640142 3014 leader_election.cc:288] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f6cda9ea914a448ebe7d4c25dec6364f (127.2.108.62:39471), 6fba5a8478374b1fb2c7b5a8ff282d2b (127.2.108.61:33947)
23:31:19.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:19.642221 2793 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.108.62:39471: connect: Connection refused (error 111) [suppressed 3 similar messages]
23:31:19.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:19.655745 2791 leader_election.cc:334] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 6fba5a8478374b1fb2c7b5a8ff282d2b (127.2.108.61:33947): Network error: Client connection negotiation failed: client connection to 127.2.108.61:33947: connect: Connection refused (error 111)
23:31:19.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:19.656610 2793 leader_election.cc:334] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer f6cda9ea914a448ebe7d4c25dec6364f (127.2.108.62:39471): Network error: Client connection negotiation failed: client connection to 127.2.108.62:39471: connect: Connection refused (error 111)
23:31:19.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:19.657045 2793 leader_election.cc:302] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5739a890599e4735b0d4529f0064cdc2; no voters: 6fba5a8478374b1fb2c7b5a8ff282d2b, f6cda9ea914a448ebe7d4c25dec6364f
23:31:19.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:19.657867 3014 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
23:31:19.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:19.879778 2856 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:31:19.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:19.880013 2856 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:31:19.970 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.108.62:39471(127.2.108.62:39471)] connecting to peer
23:31:19.971 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.108.61:33947(127.2.108.61:33947)] connecting to peer
23:31:19.972 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471
23:31:19.973 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.108.62:39471(127.2.108.62:39471)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471
23:31:19.973 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.108.62:39471: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471
23:31:19.974 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
23:31:19.974 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.108.61:33947(127.2.108.61:33947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
23:31:19.974 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.108.61:33947: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
23:31:19.974 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.2.108.60:42421,127.2.108.62:39471,127.2.108.61:33947; will retry
23:31:20.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:20.882899 2856 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:31:20.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:20.883234 2856 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:31:21.809 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.108.62:39471(127.2.108.62:39471)] connecting to peer
23:31:21.811 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.108.61:33947(127.2.108.61:33947)] connecting to peer
23:31:21.811 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471
23:31:21.812 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.108.62:39471(127.2.108.62:39471)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471
23:31:21.812 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.108.62:39471: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471
23:31:21.815 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
23:31:21.815 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.108.61:33947(127.2.108.61:33947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
23:31:21.815 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.108.61:33947: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
23:31:21.816 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.2.108.60:42421,127.2.108.62:39471,127.2.108.61:33947; will retry
23:31:21.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:21.886524 2856 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:31:21.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:21.886823 2856 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:31:22.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:22.890743 2856 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:31:22.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:22.891099 2856 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:31:23.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 23:31:23 dist-test-slave-dnlc krb5kdc[2493](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790551883, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.108.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
23:31:23.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:23.578254 2785 init.cc:295] Successfully reacquired a new kerberos TGT
23:31:23.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:23.824900 3024 raft_consensus.cc:482] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
23:31:23.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:23.825429 3024 raft_consensus.cc:504] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f6cda9ea914a448ebe7d4c25dec6364f" member_type: VOTER last_known_addr { host: "127.2.108.62" port: 39471 } } peers { permanent_uuid: "6fba5a8478374b1fb2c7b5a8ff282d2b" member_type: VOTER last_known_addr { host: "127.2.108.61" port: 33947 } } peers { permanent_uuid: "5739a890599e4735b0d4529f0064cdc2" member_type: VOTER last_known_addr { host: "127.2.108.60" port: 42421 } }
23:31:23.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:23.827409 3024 leader_election.cc:288] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f6cda9ea914a448ebe7d4c25dec6364f (127.2.108.62:39471), 6fba5a8478374b1fb2c7b5a8ff282d2b (127.2.108.61:33947)
23:31:23.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:23.832851 2793 leader_election.cc:334] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer f6cda9ea914a448ebe7d4c25dec6364f (127.2.108.62:39471): Network error: Client connection negotiation failed: client connection to 127.2.108.62:39471: connect: Connection refused (error 111)
23:31:23.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:23.833803 2791 leader_election.cc:334] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 6fba5a8478374b1fb2c7b5a8ff282d2b (127.2.108.61:33947): Network error: Client connection negotiation failed: client connection to 127.2.108.61:33947: connect: Connection refused (error 111)
23:31:23.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:23.834178 2791 leader_election.cc:302] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5739a890599e4735b0d4529f0064cdc2; no voters: 6fba5a8478374b1fb2c7b5a8ff282d2b, f6cda9ea914a448ebe7d4c25dec6364f
23:31:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:23.834832 3024 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
23:31:23.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:23.894425 2856 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:31:23.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:23.894693 2856 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:31:23.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:23.912583 2768 master_runner.cc:186] Error getting master registration for 127.2.108.62:39471: Network error: Client connection negotiation failed: client connection to 127.2.108.62:39471: connect: Connection refused (error 111),
23:31:23.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:23.913003 2768 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
23:31:24.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:24.897485 2856 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:31:24.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:24.897785 2856 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:31:24.969 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.108.62:39471(127.2.108.62:39471)] connecting to peer
23:31:24.970 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471
23:31:24.971 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.108.62:39471(127.2.108.62:39471)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471
23:31:24.971 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.108.62:39471: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471
23:31:24.971 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.108.61:33947(127.2.108.61:33947)] connecting to peer
23:31:24.973 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
23:31:24.973 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.108.61:33947(127.2.108.61:33947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
23:31:24.973 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.108.61:33947: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
23:31:24.974 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.2.108.60:42421,127.2.108.62:39471,127.2.108.61:33947; will retry
23:31:25.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:25.901058 2856 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:31:25.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:25.901376 2856 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:31:26.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:26.904165 2856 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:31:26.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:26.904462 2856 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:31:27.069 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.108.62:39471(127.2.108.62:39471)] connecting to peer
23:31:27.071 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.108.61:33947(127.2.108.61:33947)] connecting to peer
23:31:27.071 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471
23:31:27.072 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.108.62:39471(127.2.108.62:39471)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471
23:31:27.072 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.108.62:39471: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471
23:31:27.074 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
23:31:27.074 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.108.61:33947(127.2.108.61:33947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
23:31:27.074 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.108.61:33947: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
23:31:27.074 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.2.108.60:42421,127.2.108.62:39471,127.2.108.61:33947; will retry
23:31:27.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:27.907306 2856 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:31:27.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:27.907588 2856 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:31:28.689 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.108.62:39471(127.2.108.62:39471)] connecting to peer
23:31:28.691 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.108.61:33947(127.2.108.61:33947)] connecting to peer
23:31:28.692 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471
23:31:28.692 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.108.62:39471(127.2.108.62:39471)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471
23:31:28.693 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.108.62:39471: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471
23:31:28.693 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
23:31:28.693 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.108.61:33947(127.2.108.61:33947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
23:31:28.693 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.108.61:33947: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
23:31:28.694 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.2.108.60:42421,127.2.108.62:39471,127.2.108.61:33947; will retry
23:31:28.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:28.910276 2856 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:31:28.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:28.910521 2856 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:31:28.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:28.915730 2768 master_runner.cc:186] Error getting master registration for 127.2.108.62:39471: Network error: Client connection negotiation failed: client connection to 127.2.108.62:39471: connect: Connection refused (error 111),
23:31:28.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:28.916095 2768 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
23:31:29.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:29.913360 2856 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:31:29.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:29.913688 2856 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:31:30.169 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.108.62:39471(127.2.108.62:39471)] connecting to peer
23:31:30.170 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.108.61:33947(127.2.108.61:33947)] connecting to peer
23:31:30.172 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471
23:31:30.172 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.108.62:39471(127.2.108.62:39471)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471
23:31:30.173 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.108.62:39471: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471
23:31:30.173 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
23:31:30.173 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.108.61:33947(127.2.108.61:33947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
23:31:30.173 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.108.61:33947: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
23:31:30.174 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.2.108.60:42421,127.2.108.62:39471,127.2.108.61:33947; will retry
23:31:30.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:30.916788 2856 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:31:30.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:30.917026 2856 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:31:31.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:31.394073 3038 raft_consensus.cc:482] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
23:31:31.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:31.394439 3038 raft_consensus.cc:504] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f6cda9ea914a448ebe7d4c25dec6364f" member_type: VOTER last_known_addr { host: "127.2.108.62" port: 39471 } } peers { permanent_uuid: "6fba5a8478374b1fb2c7b5a8ff282d2b" member_type: VOTER last_known_addr { host: "127.2.108.61" port: 33947 } } peers { permanent_uuid: "5739a890599e4735b0d4529f0064cdc2" member_type: VOTER last_known_addr { host: "127.2.108.60" port: 42421 } }
23:31:31.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:31.395936 3038 leader_election.cc:288] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f6cda9ea914a448ebe7d4c25dec6364f (127.2.108.62:39471), 6fba5a8478374b1fb2c7b5a8ff282d2b (127.2.108.61:33947)
23:31:31.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:31.398120 2793 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.108.62:39471: connect: Connection refused (error 111) [suppressed 3 similar messages]
23:31:31.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:31.400713 2791 leader_election.cc:334] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 6fba5a8478374b1fb2c7b5a8ff282d2b (127.2.108.61:33947): Network error: Client connection negotiation failed: client connection to 127.2.108.61:33947: connect: Connection refused (error 111)
23:31:31.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:31.401155 2793 leader_election.cc:334] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer f6cda9ea914a448ebe7d4c25dec6364f (127.2.108.62:39471): Network error: Client connection negotiation failed: client connection to 127.2.108.62:39471: connect: Connection refused (error 111)
23:31:31.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:31.401530 2793 leader_election.cc:302] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5739a890599e4735b0d4529f0064cdc2; no voters: 6fba5a8478374b1fb2c7b5a8ff282d2b, f6cda9ea914a448ebe7d4c25dec6364f
23:31:31.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:31.402251 3038 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
23:31:31.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:31.919767 2856 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:31:31.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:31.920207 2856 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:31:32.209 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.108.62:39471(127.2.108.62:39471)] connecting to peer
23:31:32.210 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.108.61:33947(127.2.108.61:33947)] connecting to peer
23:31:32.211 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471
23:31:32.211 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.108.62:39471(127.2.108.62:39471)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471
23:31:32.211 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.108.62:39471: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471
23:31:32.213 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
23:31:32.213 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.108.61:33947(127.2.108.61:33947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
23:31:32.213 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.108.61:33947: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
23:31:32.214 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.2.108.60:42421,127.2.108.62:39471,127.2.108.61:33947; will retry
23:31:32.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:32.923571 2856 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:31:32.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:32.923894 2856 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:31:33.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 23:31:33 dist-test-slave-dnlc krb5kdc[2493](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790551893, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.108.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
23:31:33.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:33.580933 2785 init.cc:295] Successfully reacquired a new kerberos TGT
23:31:33.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:33.919335 2768 master_runner.cc:186] Error getting master registration for 127.2.108.62:39471: Network error: Client connection negotiation failed: client connection to 127.2.108.62:39471: connect: Connection refused (error 111),
23:31:33.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:33.919811 2768 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
23:31:33.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:33.927023 2856 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:31:33.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:33.927290 2856 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:31:34.509 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.108.62:39471(127.2.108.62:39471)] connecting to peer
23:31:34.511 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471
23:31:34.512 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.108.62:39471(127.2.108.62:39471)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471
23:31:34.512 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.108.62:39471: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471
23:31:34.512 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.108.61:33947(127.2.108.61:33947)] connecting to peer
23:31:34.514 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
23:31:34.515 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.108.61:33947(127.2.108.61:33947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
23:31:34.515 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.108.61:33947: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
23:31:34.515 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.2.108.60:42421,127.2.108.62:39471,127.2.108.61:33947; will retry
23:31:34.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:34.930205 2856 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:31:34.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:34.930480 2856 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:31:35.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:35.933174 2856 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:31:35.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:35.933470 2856 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:31:36.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:36.936209 2856 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:31:36.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:36.936494 2856 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:31:37.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:37.267789 1827 debug-util.cc:398] Leaking SignalData structure 0x7b08000a5820 after lost signal to thread 1764
23:31:37.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:37.268543 1827 debug-util.cc:398] Leaking SignalData structure 0x7b08000b02e0 after lost signal to thread 1830
23:31:37.850 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.108.62:39471(127.2.108.62:39471)] connecting to peer
23:31:37.850 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.108.61:33947(127.2.108.61:33947)] connecting to peer
23:31:37.852 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471
23:31:37.852 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.108.62:39471(127.2.108.62:39471)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471
23:31:37.852 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.108.62:39471: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471
23:31:37.855 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
23:31:37.855 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.108.61:33947(127.2.108.61:33947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
23:31:37.855 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.108.61:33947: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
23:31:37.855 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.2.108.60:42421,127.2.108.62:39471,127.2.108.61:33947; will retry
23:31:37.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:37.939327 2856 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:31:37.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:37.939630 2856 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:31:38.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:38.922538 2768 master_runner.cc:186] Error getting master registration for 127.2.108.62:39471: Network error: Client connection negotiation failed: client connection to 127.2.108.62:39471: connect: Connection refused (error 111),
23:31:38.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:38.922855 2768 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
23:31:38.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:38.942422 2856 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:31:38.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:38.942660 2856 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:31:39.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:39.945430 2856 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:31:39.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:39.945806 2856 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:31:40.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:40.038118 3050 raft_consensus.cc:482] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
23:31:40.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:40.038587 3050 raft_consensus.cc:504] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f6cda9ea914a448ebe7d4c25dec6364f" member_type: VOTER last_known_addr { host: "127.2.108.62" port: 39471 } } peers { permanent_uuid: "6fba5a8478374b1fb2c7b5a8ff282d2b" member_type: VOTER last_known_addr { host: "127.2.108.61" port: 33947 } } peers { permanent_uuid: "5739a890599e4735b0d4529f0064cdc2" member_type: VOTER last_known_addr { host: "127.2.108.60" port: 42421 } }
23:31:40.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:40.040567 3050 leader_election.cc:288] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f6cda9ea914a448ebe7d4c25dec6364f (127.2.108.62:39471), 6fba5a8478374b1fb2c7b5a8ff282d2b (127.2.108.61:33947)
23:31:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:40.042822 2793 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.108.62:39471: connect: Connection refused (error 111) [suppressed 1 similar messages]
23:31:40.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:40.045884 2793 leader_election.cc:334] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer f6cda9ea914a448ebe7d4c25dec6364f (127.2.108.62:39471): Network error: Client connection negotiation failed: client connection to 127.2.108.62:39471: connect: Connection refused (error 111)
23:31:40.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:40.046407 2791 leader_election.cc:334] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 6fba5a8478374b1fb2c7b5a8ff282d2b (127.2.108.61:33947): Network error: Client connection negotiation failed: client connection to 127.2.108.61:33947: connect: Connection refused (error 111)
23:31:40.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:40.046725 2791 leader_election.cc:302] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5739a890599e4735b0d4529f0064cdc2; no voters: 6fba5a8478374b1fb2c7b5a8ff282d2b, f6cda9ea914a448ebe7d4c25dec6364f
23:31:40.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:40.047242 3050 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
23:31:40.269 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.108.62:39471(127.2.108.62:39471)] connecting to peer
23:31:40.270 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.108.61:33947(127.2.108.61:33947)] connecting to peer
23:31:40.272 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471
23:31:40.272 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.108.62:39471(127.2.108.62:39471)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471
23:31:40.272 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
23:31:40.272 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.108.61:33947(127.2.108.61:33947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
23:31:40.272 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.108.62:39471: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471
23:31:40.272 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.108.61:33947: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
23:31:40.273 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.2.108.60:42421,127.2.108.62:39471,127.2.108.61:33947; will retry
23:31:40.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:40.948535 2856 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:31:40.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:40.948778 2856 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:31:41.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:41.951555 2856 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:31:41.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:41.951802 2856 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:31:42.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:42.956154 2856 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:31:42.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:42.956545 2856 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:31:43.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 23:31:43 dist-test-slave-dnlc krb5kdc[2493](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790551903, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.108.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
23:31:43.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:43.583385 2785 init.cc:295] Successfully reacquired a new kerberos TGT
23:31:43.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:43.925912 2768 master_runner.cc:186] Error getting master registration for 127.2.108.62:39471: Network error: Client connection negotiation failed: client connection to 127.2.108.62:39471: connect: Connection refused (error 111),
23:31:43.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:43.926344 2768 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
23:31:43.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:43.959704 2856 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:31:43.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:43.959950 2856 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:31:44.109 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.108.62:39471(127.2.108.62:39471)] connecting to peer
23:31:44.109 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.108.61:33947(127.2.108.61:33947)] connecting to peer
23:31:44.110 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471
23:31:44.110 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.108.62:39471(127.2.108.62:39471)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471
23:31:44.110 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
23:31:44.111 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.108.61:33947(127.2.108.61:33947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
23:31:44.111 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.108.62:39471: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471
23:31:44.111 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.108.61:33947: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
23:31:44.113 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.2.108.60:42421,127.2.108.62:39471,127.2.108.61:33947; will retry
23:31:44.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:44.962890 2856 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:31:44.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:44.963127 2856 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:31:45.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:45.965572 2856 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:31:45.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:45.965824 2856 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:31:46.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:46.968181 2856 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5739a890599e4735b0d4529f0064cdc2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:31:46.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 23:31:46.968410 2856 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:31:47.930 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.108.62:39471(127.2.108.62:39471)] connecting to peer
23:31:47.930 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.2.108.61:33947(127.2.108.61:33947)] connecting to peer
23:31:47.931 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471
23:31:47.931 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.108.62:39471(127.2.108.62:39471)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471
23:31:47.931 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
23:31:47.931 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.108.62:39471: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471
23:31:47.931 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.108.61:33947(127.2.108.61:33947)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
23:31:47.931 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.108.61:33947: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
23:31:47.933 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.2.108.60:42421,127.2.108.62:39471,127.2.108.61:33947; will retry
23:31:47.936 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=22, TimeoutTracker(timeout=30000, elapsed=29945), Traces: [0ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.60:42421, [137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.62:39471, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.61:33947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.62:39471: Network error: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471, [242ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.61:33947: Network error: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [554ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.60:42421: OK, [558ms] delaying RPC due to: Service unavailable: Master config (127.2.108.60:42421,127.2.108.62:39471,127.2.108.61:33947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [575ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.60:42421, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.62:39471, [579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.61:33947, [583ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.62:39471: Network error: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471, [586ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.60:42421: OK, [587ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.61:33947: Network error: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [588ms] delaying RPC due to: Service unavailable: Master config (127.2.108.60:42421,127.2.108.62:39471,127.2.108.61:33947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [596ms] refreshing cache from master, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.60:42421, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.62:39471, [600ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.61:33947, [601ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.60:42421: OK, [601ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.62:39471: Network error: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471, [604ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.61:33947: Network error: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [605ms] delaying RPC due to: Service unavailable: Master config (127.2.108.60:42421,127.2.108.62:39471,127.2.108.61:33947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [615ms] refreshing cache from master, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.60:42421, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.62:39471, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.61:33947, [619ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.62:39471: Network error: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471, [622ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.60:42421: OK, [624ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.61:33947: Network error: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [627ms] delaying RPC due to: Service unavailable: Master config (127.2.108.60:42421,127.2.108.62:39471,127.2.108.61:33947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [655ms] refreshing cache from master, [655ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.60:42421, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.62:39471, [661ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.60:42421: OK, [662ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.61:33947, [663ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.62:39471: Network error: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471, [665ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.61:33947: Network error: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [666ms] delaying RPC due to: Service unavailable: Master config (127.2.108.60:42421,127.2.108.62:39471,127.2.108.61:33947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [696ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.60:42421, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.62:39471, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.61:33947, [700ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.62:39471: Network error: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471, [701ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.60:42421: OK, [712ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.61:33947: Network error: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [713ms] delaying RPC due to: Service unavailable: Master config (127.2.108.60:42421,127.2.108.62:39471,127.2.108.61:33947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [735ms] refreshing cache from master, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.60:42421, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.62:39471, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.61:33947, [739ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.62:39471: Network error: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471, [740ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.60:42421: OK, [742ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.61:33947: Network error: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [743ms] delaying RPC due to: Service unavailable: Master config (127.2.108.60:42421,127.2.108.62:39471,127.2.108.61:33947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [856ms] refreshing cache from master, [856ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.60:42421, [857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.62:39471, [859ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.61:33947, [862ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.60:42421: OK, [864ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.62:39471: Network error: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471, [864ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.61:33947: Network error: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [866ms] delaying RPC due to: Service unavailable: Master config (127.2.108.60:42421,127.2.108.62:39471,127.2.108.61:33947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [976ms] refreshing cache from master, [976ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.60:42421, [976ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.62:39471, [977ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.61:33947, [980ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.62:39471: Network error: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471, [981ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.60:42421: OK, [982ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.61:33947: Network error: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [983ms] delaying RPC due to: Service unavailable: Master config (127.2.108.60:42421,127.2.108.62:39471,127.2.108.61:33947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [1116ms] refreshing cache from master, [1116ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.60:42421, [1116ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.62:39471, [1119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.61:33947, [1120ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.62:39471: Network error: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471, [1121ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.60:42421: OK, [1132ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.61:33947: Network error: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [1134ms] delaying RPC due to: Service unavailable: Master config (127.2.108.60:42421,127.2.108.62:39471,127.2.108.61:33947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [1976ms] refreshing cache from master, [1976ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.60:42421, [1976ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.62:39471, [1978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.61:33947, [1980ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.62:39471: Network error: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471, [1981ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.60:42421: OK, [1981ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.61:33947: Network error: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [1981ms] delaying RPC due to: Service unavailable: Master config (127.2.108.60:42421,127.2.108.62:39471,127.2.108.61:33947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [3815ms] refreshing cache from master, [3815ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.60:42421, [3816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.62:39471, [3817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.61:33947, [3819ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.62:39471: Network error: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471, [3820ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.60:42421: OK, [3822ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.61:33947: Network error: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [3823ms] delaying RPC due to: Service unavailable: Master config (127.2.108.60:42421,127.2.108.62:39471,127.2.108.61:33947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [6975ms] refreshing cache from master, [6975ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.60:42421, [6975ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.62:39471, [6978ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.62:39471: Network error: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471, [6978ms] too many traces: truncated at 100 traces, deferred=Deferred@318451981(state=PENDING, result=null, callback=(continuation of Deferred@1276053814 after retry RPC after error@122644415) -> (continuation of Deferred@1208818495 after retry RPC after error@53815402) -> (continuation of Deferred@935839619 after retry RPC after error@1263086683) -> (continuation of Deferred@1799160558 after retry RPC after error@1096876443) -> (continuation of Deferred@1217605099 after retry RPC after error@1245379414) -> (continuation of Deferred@1234418317 after retry RPC after error@11995133) -> (continuation of Deferred@1369083509 after retry RPC after error@1281834843) -> (continuation of Deferred@418689928 after retry RPC after error@549771140) -> (continuation of Deferred@623381711 after retry RPC after error@1346396886) -> (continuation of Deferred@1838543781 after retry RPC after error@1802199396) -> (continuation of Deferred@247486130 after retry RPC after error@1573241710) -> (continuation of Deferred@529468328 after retry RPC after error@803158396) -> (continuation of Deferred@589733875 after retry RPC after error@1342961226) -> (continuation of Deferred@926840232 after retry RPC after error@1635565290) -> (continuation of Deferred@1991404593 after retry RPC after error@2038725899) -> (continuation of Deferred@1037288614 after retry RPC after error@970761752) -> (continuation of Deferred@1558873846 after retry RPC after error@106742853) -> (continuation of Deferred@621401820 after retry RPC after error@2042189827) -> (continuation of Deferred@1902951668 after retry RPC after error@292692080) -> (continuation of Deferred@2057743284 after retry RPC after error@1231855011) -> (continuation of Deferred@1576211302 after retry RPC after error@1560467720), errback=(continuation of Deferred@1276053814 after retry RPC after error@122644415) -> (continuation of Deferred@1208818495 after retry RPC after error@53815402) -> (continuation of Deferred@935839619 after retry RPC after error@1263086683) -> (continuation of Deferred@1799160558 after retry RPC after error@1096876443) -> (continuation of Deferred@1217605099 after retry RPC after error@1245379414) -> (continuation of Deferred@1234418317 after retry RPC after error@11995133) -> (continuation of Deferred@1369083509 after retry RPC after error@1281834843) -> (continuation of Deferred@418689928 after retry RPC after error@549771140) -> (continuation of Deferred@623381711 after retry RPC after error@1346396886) -> (continuation of Deferred@1838543781 after retry RPC after error@1802199396) -> (continuation of Deferred@247486130 after retry RPC after error@1573241710) -> (continuation of Deferred@529468328 after retry RPC after error@803158396) -> (continuation of Deferred@589733875 after retry RPC after error@1342961226) -> (continuation of Deferred@926840232 after retry RPC after error@1635565290) -> (continuation of Deferred@1991404593 after retry RPC after error@2038725899) -> (continuation of Deferred@1037288614 after retry RPC after error@970761752) -> (continuation of Deferred@1558873846 after retry RPC after error@106742853) -> (continuation of Deferred@621401820 after retry RPC after error@2042189827) -> (continuation of Deferred@1902951668 after retry RPC after error@292692080) -> (continuation of Deferred@2057743284 after retry RPC after error@1231855011) -> (continuation of Deferred@1576211302 after retry RPC after error@1560467720)))
23:31:47.941 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.2.108.60:42421(127.2.108.60:42421)] disconnecting while in state READY
23:31:47.942 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.2.108.61:33947(127.2.108.61:33947)] disconnecting while in state TERMINATED
23:31:47.942 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.2.108.62:39471(127.2.108.62:39471)] disconnecting while in state TERMINATED
23:31:47.946 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
23:31:47.948 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.2.108.60:42421(127.2.108.60:42421)] handling channelInactive
23:31:47.948 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.108.60:42421(127.2.108.60:42421)] cleaning up while in state READY due to: connection closed
23:31:47.950 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
23:31:47.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:47.958207 2480 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskpPnkEK/build/tsan/bin/kudu with pid 2768
23:31:49.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T23:31:49Z chronyd exiting
23:31:50.170 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
23:31:50.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:50.172043 1751 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskpPnkEK/build/tsan/bin/kudu with pid 2032
23:31:50.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:50.196048 1751 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskpPnkEK/build/tsan/bin/kudu with pid 2195
23:31:50.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:50.221381 1751 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskpPnkEK/build/tsan/bin/kudu with pid 2339
23:31:50.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:50.250458 1751 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskpPnkEK/build/tsan/bin/kudu with pid 1763
23:31:50.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:50.280036 1751 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskpPnkEK/build/tsan/bin/kudu with pid 1833
23:31:50.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 23:31:50.310950 1751 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskpPnkEK/build/tsan/bin/kudu with pid 1908
23:31:51.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T23:31:51Z chronyd exiting
23:31:52.477 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=22, TimeoutTracker(timeout=30000, elapsed=29945), Traces: [0ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.60:42421, [137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.62:39471, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.61:33947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.62:39471: Network error: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471, [242ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.61:33947: Network error: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [554ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.60:42421: OK, [558ms] delaying RPC due to: Service unavailable: Master config (127.2.108.60:42421,127.2.108.62:39471,127.2.108.61:33947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [575ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.60:42421, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.62:39471, [579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.61:33947, [583ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.62:39471: Network error: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471, [586ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.60:42421: OK, [587ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.61:33947: Network error: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [588ms] delaying RPC due to: Service unavailable: Master config (127.2.108.60:42421,127.2.108.62:39471,127.2.108.61:33947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [596ms] refreshing cache from master, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.60:42421, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.62:39471, [600ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.61:33947, [601ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.60:42421: OK, [601ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.62:39471: Network error: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471, [604ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.61:33947: Network error: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [605ms] delaying RPC due to: Service unavailable: Master config (127.2.108.60:42421,127.2.108.62:39471,127.2.108.61:33947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [615ms] refreshing cache from master, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.60:42421, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.62:39471, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.61:33947, [619ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.62:39471: Network error: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471, [622ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.60:42421: OK, [624ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.61:33947: Network error: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [627ms] delaying RPC due to: Service unavailable: Master config (127.2.108.60:42421,127.2.108.62:39471,127.2.108.61:33947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [655ms] refreshing cache from master, [655ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.60:42421, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.62:39471, [661ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.60:42421: OK, [662ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.61:33947, [663ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.62:39471: Network error: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471, [665ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.61:33947: Network error: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [666ms] delaying RPC due to: Service unavailable: Master config (127.2.108.60:42421,127.2.108.62:39471,127.2.108.61:33947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [696ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.60:42421, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.62:39471, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.61:33947, [700ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.62:39471: Network error: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471, [701ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.60:42421: OK, [712ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.61:33947: Network error: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [713ms] delaying RPC due to: Service unavailable: Master config (127.2.108.60:42421,127.2.108.62:39471,127.2.108.61:33947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [735ms] refreshing cache from master, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.60:42421, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.62:39471, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.61:33947, [739ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.62:39471: Network error: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471, [740ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.60:42421: OK, [742ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.61:33947: Network error: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [743ms] delaying RPC due to: Service unavailable: Master config (127.2.108.60:42421,127.2.108.62:39471,127.2.108.61:33947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [856ms] refreshing cache from master, [856ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.60:42421, [857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.62:39471, [859ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.61:33947, [862ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.60:42421: OK, [864ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.62:39471: Network error: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471, [864ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.61:33947: Network error: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [866ms] delaying RPC due to: Service unavailable: Master config (127.2.108.60:42421,127.2.108.62:39471,127.2.108.61:33947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [976ms] refreshing cache from master, [976ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.60:42421, [976ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.62:39471, [977ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.61:33947, [980ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.62:39471: Network error: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471, [981ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.60:42421: OK, [982ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.61:33947: Network error: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [983ms] delaying RPC due to: Service unavailable: Master config (127.2.108.60:42421,127.2.108.62:39471,127.2.108.61:33947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [1116ms] refreshing cache from master, [1116ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.60:42421, [1116ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.62:39471, [1119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.61:33947, [1120ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.62:39471: Network error: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471, [1121ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.60:42421: OK, [1132ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.61:33947: Network error: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [1134ms] delaying RPC due to: Service unavailable: Master config (127.2.108.60:42421,127.2.108.62:39471,127.2.108.61:33947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [1976ms] refreshing cache from master, [1976ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.60:42421, [1976ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.62:39471, [1978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.61:33947, [1980ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.62:39471: Network error: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471, [1981ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.60:42421: OK, [1981ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.61:33947: Network error: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [1981ms] delaying RPC due to: Service unavailable: Master config (127.2.108.60:42421,127.2.108.62:39471,127.2.108.61:33947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [3815ms] refreshing cache from master, [3815ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.60:42421, [3816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.62:39471, [3817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.61:33947, [3819ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.62:39471: Network error: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471, [3820ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.60:42421: OK, [3822ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.61:33947: Network error: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [3823ms] delaying RPC due to: Service unavailable: Master config (127.2.108.60:42421,127.2.108.62:39471,127.2.108.61:33947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [6975ms] refreshing cache from master, [6975ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.60:42421, [6975ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.62:39471, [6978ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.62:39471: Network error: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471, [6978ms] too many traces: truncated at 100 traces, deferred=Deferred@318451981(state=PENDING, result=null, callback=(continuation of Deferred@1276053814 after retry RPC after error@122644415) -> (continuation of Deferred@1208818495 after retry RPC after error@53815402) -> (continuation of Deferred@935839619 after retry RPC after error@1263086683) -> (continuation of Deferred@1799160558 after retry RPC after error@1096876443) -> (continuation of Deferred@1217605099 after retry RPC after error@1245379414) -> (continuation of Deferred@1234418317 after retry RPC after error@11995133) -> (continuation of Deferred@1369083509 after retry RPC after error@1281834843) -> (continuation of Deferred@418689928 after retry RPC after error@549771140) -> (continuation of Deferred@623381711 after retry RPC after error@1346396886) -> (continuation of Deferred@1838543781 after retry RPC after error@1802199396) -> (continuation of Deferred@247486130 after retry RPC after error@1573241710) -> (continuation of Deferred@529468328 after retry RPC after error@803158396) -> (continuation of Deferred@589733875 after retry RPC after error@1342961226) -> (continuation of Deferred@926840232 after retry RPC after error@1635565290) -> (continuation of Deferred@1991404593 after retry RPC after error@2038725899) -> (continuation of Deferred@1037288614 after retry RPC after error@970761752) -> (continuation of Deferred@1558873846 after retry RPC after error@106742853) -> (continuation of Deferred@621401820 after retry RPC after error@2042189827) -> (continuation of Deferred@1902951668 after retry RPC after error@292692080) -> (continuation of Deferred@2057743284 after retry RPC after error@1231855011) -> (continuation of Deferred@1576211302 after retry RPC after error@1560467720), errback=(continuation of Deferred@1276053814 after retry RPC after error@122644415) -> (continuation of Deferred@1208818495 after retry RPC after error@53815402) -> (continuation of Deferred@935839619 after retry RPC after error@1263086683) -> (continuation of Deferred@1799160558 after retry RPC after error@1096876443) -> (continuation of Deferred@1217605099 after retry RPC after error@1245379414) -> (continuation of Deferred@1234418317 after retry RPC after error@11995133) -> (continuation of Deferred@1369083509 after retry RPC after error@1281834843) -> (continuation of Deferred@418689928 after retry RPC after error@549771140) -> (continuation of Deferred@623381711 after retry RPC after error@1346396886) -> (continuation of Deferred@1838543781 after retry RPC after error@1802199396) -> (continuation of Deferred@247486130 after retry RPC after error@1573241710) -> (continuation of Deferred@529468328 after retry RPC after error@803158396) -> (continuation of Deferred@589733875 after retry RPC after error@1342961226) -> (continuation of Deferred@926840232 after retry RPC after error@1635565290) -> (continuation of Deferred@1991404593 after retry RPC after error@2038725899) -> (continuation of Deferred@1037288614 after retry RPC after error@970761752) -> (continuation of Deferred@1558873846 after retry RPC after error@106742853) -> (continuation of Deferred@621401820 after retry RPC after error@2042189827) -> (continuation of Deferred@1902951668 after retry RPC after error@292692080) -> (continuation of Deferred@2057743284 after retry RPC after error@1231855011) -> (continuation of Deferred@1576211302 after retry RPC after error@1560467720)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.2.108.60:42421,127.2.108.62:39471,127.2.108.61:33947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.2.108.61:33947
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=22, TimeoutTracker(timeout=30000, elapsed=29945), Traces: [0ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.60:42421, [137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.62:39471, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.61:33947, [182ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.62:39471: Network error: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471, [242ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.61:33947: Network error: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [554ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.60:42421: OK, [558ms] delaying RPC due to: Service unavailable: Master config (127.2.108.60:42421,127.2.108.62:39471,127.2.108.61:33947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [575ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.60:42421, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.62:39471, [579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.61:33947, [583ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.62:39471: Network error: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471, [586ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.60:42421: OK, [587ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.61:33947: Network error: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [588ms] delaying RPC due to: Service unavailable: Master config (127.2.108.60:42421,127.2.108.62:39471,127.2.108.61:33947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [596ms] refreshing cache from master, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.60:42421, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.62:39471, [600ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.61:33947, [601ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.60:42421: OK, [601ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.62:39471: Network error: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471, [604ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.61:33947: Network error: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [605ms] delaying RPC due to: Service unavailable: Master config (127.2.108.60:42421,127.2.108.62:39471,127.2.108.61:33947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [615ms] refreshing cache from master, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.60:42421, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.62:39471, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.61:33947, [619ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.62:39471: Network error: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471, [622ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.60:42421: OK, [624ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.61:33947: Network error: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [627ms] delaying RPC due to: Service unavailable: Master config (127.2.108.60:42421,127.2.108.62:39471,127.2.108.61:33947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [655ms] refreshing cache from master, [655ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.60:42421, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.62:39471, [661ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.60:42421: OK, [662ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.61:33947, [663ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.62:39471: Network error: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471, [665ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.61:33947: Network error: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [666ms] delaying RPC due to: Service unavailable: Master config (127.2.108.60:42421,127.2.108.62:39471,127.2.108.61:33947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [696ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.60:42421, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.62:39471, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.61:33947, [700ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.62:39471: Network error: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471, [701ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.60:42421: OK, [712ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.61:33947: Network error: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [713ms] delaying RPC due to: Service unavailable: Master config (127.2.108.60:42421,127.2.108.62:39471,127.2.108.61:33947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [735ms] refreshing cache from master, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.60:42421, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.62:39471, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.61:33947, [739ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.62:39471: Network error: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471, [740ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.60:42421: OK, [742ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.61:33947: Network error: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [743ms] delaying RPC due to: Service unavailable: Master config (127.2.108.60:42421,127.2.108.62:39471,127.2.108.61:33947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [856ms] refreshing cache from master, [856ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.60:42421, [857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.62:39471, [859ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.61:33947, [862ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.60:42421: OK, [864ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.62:39471: Network error: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471, [864ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.61:33947: Network error: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [866ms] delaying RPC due to: Service unavailable: Master config (127.2.108.60:42421,127.2.108.62:39471,127.2.108.61:33947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [976ms] refreshing cache from master, [976ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.60:42421, [976ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.62:39471, [977ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.61:33947, [980ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.62:39471: Network error: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471, [981ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.60:42421: OK, [982ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.61:33947: Network error: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [983ms] delaying RPC due to: Service unavailable: Master config (127.2.108.60:42421,127.2.108.62:39471,127.2.108.61:33947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [1116ms] refreshing cache from master, [1116ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.60:42421, [1116ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.62:39471, [1119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.61:33947, [1120ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.62:39471: Network error: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471, [1121ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.60:42421: OK, [1132ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.61:33947: Network error: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [1134ms] delaying RPC due to: Service unavailable: Master config (127.2.108.60:42421,127.2.108.62:39471,127.2.108.61:33947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [1976ms] refreshing cache from master, [1976ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.60:42421, [1976ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.62:39471, [1978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.61:33947, [1980ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.62:39471: Network error: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471, [1981ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.60:42421: OK, [1981ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.61:33947: Network error: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [1981ms] delaying RPC due to: Service unavailable: Master config (127.2.108.60:42421,127.2.108.62:39471,127.2.108.61:33947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [3815ms] refreshing cache from master, [3815ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.60:42421, [3816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.62:39471, [3817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.61:33947, [3819ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.62:39471: Network error: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471, [3820ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.60:42421: OK, [3822ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.61:33947: Network error: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [3823ms] delaying RPC due to: Service unavailable: Master config (127.2.108.60:42421,127.2.108.62:39471,127.2.108.61:33947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947, [6975ms] refreshing cache from master, [6975ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.60:42421, [6975ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.108.62:39471, [6978ms] Sub RPC ConnectToMaster: received response from server master-127.2.108.62:39471: Network error: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471, [6978ms] too many traces: truncated at 100 traces, deferred=Deferred@318451981(state=PENDING, result=null, callback=(continuation of Deferred@1276053814 after retry RPC after error@122644415) -> (continuation of Deferred@1208818495 after retry RPC after error@53815402) -> (continuation of Deferred@935839619 after retry RPC after error@1263086683) -> (continuation of Deferred@1799160558 after retry RPC after error@1096876443) -> (continuation of Deferred@1217605099 after retry RPC after error@1245379414) -> (continuation of Deferred@1234418317 after retry RPC after error@11995133) -> (continuation of Deferred@1369083509 after retry RPC after error@1281834843) -> (continuation of Deferred@418689928 after retry RPC after error@549771140) -> (continuation of Deferred@623381711 after retry RPC after error@1346396886) -> (continuation of Deferred@1838543781 after retry RPC after error@1802199396) -> (continuation of Deferred@247486130 after retry RPC after error@1573241710) -> (continuation of Deferred@529468328 after retry RPC after error@803158396) -> (continuation of Deferred@589733875 after retry RPC after error@1342961226) -> (continuation of Deferred@926840232 after retry RPC after error@1635565290) -> (continuation of Deferred@1991404593 after retry RPC after error@2038725899) -> (continuation of Deferred@1037288614 after retry RPC after error@970761752) -> (continuation of Deferred@1558873846 after retry RPC after error@106742853) -> (continuation of Deferred@621401820 after retry RPC after error@2042189827) -> (continuation of Deferred@1902951668 after retry RPC after error@292692080) -> (continuation of Deferred@2057743284 after retry RPC after error@1231855011) -> (continuation of Deferred@1576211302 after retry RPC after error@1560467720), errback=(continuation of Deferred@1276053814 after retry RPC after error@122644415) -> (continuation of Deferred@1208818495 after retry RPC after error@53815402) -> (continuation of Deferred@935839619 after retry RPC after error@1263086683) -> (continuation of Deferred@1799160558 after retry RPC after error@1096876443) -> (continuation of Deferred@1217605099 after retry RPC after error@1245379414) -> (continuation of Deferred@1234418317 after retry RPC after error@11995133) -> (continuation of Deferred@1369083509 after retry RPC after error@1281834843) -> (continuation of Deferred@418689928 after retry RPC after error@549771140) -> (continuation of Deferred@623381711 after retry RPC after error@1346396886) -> (continuation of Deferred@1838543781 after retry RPC after error@1802199396) -> (continuation of Deferred@247486130 after retry RPC after error@1573241710) -> (continuation of Deferred@529468328 after retry RPC after error@803158396) -> (continuation of Deferred@589733875 after retry RPC after error@1342961226) -> (continuation of Deferred@926840232 after retry RPC after error@1635565290) -> (continuation of Deferred@1991404593 after retry RPC after error@2038725899) -> (continuation of Deferred@1037288614 after retry RPC after error@970761752) -> (continuation of Deferred@1558873846 after retry RPC after error@106742853) -> (continuation of Deferred@621401820 after retry RPC after error@2042189827) -> (continuation of Deferred@1902951668 after retry RPC after error@292692080) -> (continuation of Deferred@2057743284 after retry RPC after error@1231855011) -> (continuation of Deferred@1576211302 after retry RPC after error@1560467720)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.2.108.60:42421,127.2.108.62:39471,127.2.108.61:33947) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.62:39471(127.2.108.62:39471): Connection refused: /127.2.108.62:39471,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.108.61:33947(127.2.108.61:33947): Connection refused: /127.2.108.61:33947
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.2.108.61:33947
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)