03:50:52.535 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
03:50:52.547 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskaXaVUt/test-tmp
03:50:52.549 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
03:50:52.553 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskaXaVUt/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
03:50:52.798 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
03:50:52.951 [DEBUG - main] (MiniKuduCluster.java:195) Response:
03:50:52.952 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
03:50:52.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:52.956658 3563 mini_oidc.cc:152] Starting JWKS server
03:50:52.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:52.961395 3563 webserver.cc:460] Webserver started at http://127.0.0.1:33199/ using document root <none> and password file <none>
03:50:52.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:52.961902 3563 mini_oidc.cc:189] Starting OIDC Discovery server
03:50:52.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:52.963724 3563 webserver.cc:460] Webserver started at http://127.0.0.1:40381/ using document root <none> and password file <none>
03:50:53.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T03:50:52Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
03:50:53.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T03:50:52Z Disabled control of system clock
03:50:53.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:53.012992 3563 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskaXaVUt/build/tsan/bin/kudu
03:50:53.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskaXaVUt/build/tsan/bin/kudu
03:50:53.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/master-0/wal
03:50:53.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/master-0/data
03:50:53.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:50:53.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:50:53.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:50:53.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:50:53.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:50:53.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:50:53.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/master-0/logs
03:50:53.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/master-0/data/info.pb
03:50:53.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:50:53.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:50:53.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:50:53.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:50:53.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:50:53.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:50:53.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:50:53.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:50:53.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:50:53.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:50:53.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:50:53.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:50:53.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.122.254:42965
03:50:53.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.122.254
03:50:53.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:50:53.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.122.212:36727
03:50:53.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:50:53.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:50:53.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:50:53.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:50:53.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.122.254:42965,127.3.122.253:35383,127.3.122.252:34035
03:50:53.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:50:53.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33199/jwks.json with env {}
03:50:53.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:50:53.321511 3575 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:50:53.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:50:53.322054 3575 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:50:53.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:50:53.322688 3575 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:50:53.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:50:53.354267 3575 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:50:53.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:50:53.354557 3575 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:50:53.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:50:53.354751 3575 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:50:53.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:50:53.355027 3575 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:50:53.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:50:53.355306 3575 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:50:53.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:50:53.355517 3575 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33199/jwks.json
03:50:53.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:53.390650 3575 master_runner.cc:386] Master server non-default flags:
03:50:53.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:50:53.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.122.212:36727
03:50:53.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:50:53.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:50:53.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/master-0/data
03:50:53.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/master-0/wal
03:50:53.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:50:53.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.122.254:42965,127.3.122.253:35383,127.3.122.252:34035
03:50:53.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:50:53.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:50:53.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:50:53.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.122.254:42965
03:50:53.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:50:53.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:50:53.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:50:53.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33199/jwks.json
03:50:53.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:50:53.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:50:53.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/master-0/data/info.pb
03:50:53.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.122.254
03:50:53.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:50:53.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:50:53.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:50:53.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:50:53.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:50:53.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:50:53.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/master-0/logs
03:50:53.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:50:53.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:50:53.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:50:53.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:50:53.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:50:53.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:50:53.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:50:53.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 02:49:50 UTC on ccycloud.quasar-twsplx.root.comops.site
03:50:53.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4675617
03:50:53.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:50:53.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:53.392012 3575 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:50:53.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:53.394165 3575 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:50:53.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:50:53.413924 3584 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:50:54.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:50:53.413924 3582 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:50:54.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:53.416116 3575 server_base.cc:1034] running on GCE node
03:50:54.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:50:53.414772 3581 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:50:54.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:54.673834 3575 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:50:54.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:54.677129 3575 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:50:54.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:54.678623 3575 hybrid_clock.cc:648] HybridClock initialized: now 1790481054678571 us; error 71 us; skew 500 ppm
03:50:54.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:54.679591 3575 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:50:54.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:54.686996 3575 webserver.cc:460] Webserver started at http://127.3.122.254:41107/ using document root <none> and password file <none>
03:50:54.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:54.687917 3575 fs_manager.cc:359] Metadata directory not provided
03:50:54.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:54.688103 3575 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:50:54.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:54.688501 3575 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:50:54.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:54.693423 3575 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/master-0/data/instance:
03:50:54.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8df192a545f94500a26d100e6cefc726"
03:50:54.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 03:50:54 on dist-test-slave-c26w"
03:50:54.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:54.694563 3575 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/master-0/wal/instance:
03:50:54.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8df192a545f94500a26d100e6cefc726"
03:50:54.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 03:50:54 on dist-test-slave-c26w"
03:50:54.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:54.702116 3575 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.000s
03:50:54.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:54.707612 3591 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:50:54.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:54.708799 3575 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.000s sys 0.004s
03:50:54.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:54.709087 3575 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/master-0/data,/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/master-0/wal
03:50:54.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8df192a545f94500a26d100e6cefc726"
03:50:54.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 03:50:54 on dist-test-slave-c26w"
03:50:54.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:54.709404 3575 fs_report.cc:383] FS layout report
03:50:54.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:50:54.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/master-0/wal
03:50:54.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/master-0/wal
03:50:54.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/master-0/data/data
03:50:54.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:50:54.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:50:54.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:50:54.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:50:54.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:50:54.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:50:54.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:50:54.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:50:54.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:50:54.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:50:54.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:50:54.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:54.753480 3575 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:50:54.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:54.754906 3575 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:50:54.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:54.755316 3575 kserver.cc:163] Server-wide thread pool size limit: 3276
03:50:54.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:54.836699 3575 rpc_server.cc:307] RPC server started. Bound to: 127.3.122.254:42965
03:50:54.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:54.836874 3642 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.3.122.254:42965 every 8 connection(s)
03:50:54.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:54.839748 3575 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/master-0/data/info.pb
03:50:54.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:54.840924 3563 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskaXaVUt/build/tsan/bin/kudu as pid 3575
03:50:54.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:54.841917 3563 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/master-0/wal/instance
03:50:54.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:54.846412 3563 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskaXaVUt/build/tsan/bin/kudu
03:50:54.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskaXaVUt/build/tsan/bin/kudu
03:50:54.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/master-1/wal
03:50:54.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/master-1/data
03:50:54.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:50:54.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:50:54.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:50:54.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:50:54.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:50:54.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:50:54.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/master-1/logs
03:50:54.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/master-1/data/info.pb
03:50:54.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:50:54.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:50:54.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:50:54.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:50:54.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:50:54.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:50:54.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:50:54.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:50:54.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:50:54.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:50:54.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:50:54.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:50:54.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.122.253:35383
03:50:54.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.122.253
03:50:54.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:50:54.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.122.212:36727
03:50:54.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:50:54.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:50:54.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:50:54.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:50:54.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.122.254:42965,127.3.122.253:35383,127.3.122.252:34035
03:50:54.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:50:54.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33199/jwks.json with env {}
03:50:54.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:54.846828 3643 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:50:54.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:54.860383 3643 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.122.254" port: 42965 } has no permanent_uuid. Determining permanent_uuid...
03:50:54.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:54.895293 3643 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.122.253" port: 35383 } has no permanent_uuid. Determining permanent_uuid...
03:50:54.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:50:54.897986 3592 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.122.253:35383: connect: Connection refused (error 111)
03:50:54.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:50:54.902511 3643 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.122.253:35383: Network error: Client connection negotiation failed: client connection to 127.3.122.253:35383: connect: Connection refused (error 111)
03:50:54.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:54.952036 3643 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.122.253" port: 35383 } attempt: 1
03:50:54.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:50:54.955871 3643 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.122.253:35383: Network error: Client connection negotiation failed: client connection to 127.3.122.253:35383: connect: Connection refused (error 111)
03:50:55.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:55.024389 3643 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.122.253" port: 35383 } attempt: 2
03:50:55.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:50:55.028362 3643 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.122.253:35383: Network error: Client connection negotiation failed: client connection to 127.3.122.253:35383: connect: Connection refused (error 111)
03:50:55.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:55.120424 3643 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.122.253" port: 35383 } attempt: 3
03:50:55.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:50:55.124855 3643 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.122.253:35383: Network error: Client connection negotiation failed: client connection to 127.3.122.253:35383: connect: Connection refused (error 111)
03:50:55.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:50:55.186823 3645 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:50:55.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:50:55.187544 3645 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:50:55.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:50:55.188180 3645 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:50:55.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:50:55.218204 3645 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:50:55.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:50:55.218463 3645 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:50:55.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:50:55.218647 3645 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:50:55.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:50:55.218822 3645 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:50:55.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:50:55.219017 3645 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:50:55.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:50:55.219151 3645 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33199/jwks.json
03:50:55.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:55.253021 3645 master_runner.cc:386] Master server non-default flags:
03:50:55.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:50:55.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.122.212:36727
03:50:55.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:50:55.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:50:55.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/master-1/data
03:50:55.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/master-1/wal
03:50:55.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:50:55.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.122.254:42965,127.3.122.253:35383,127.3.122.252:34035
03:50:55.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:50:55.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:50:55.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:50:55.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.122.253:35383
03:50:55.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:50:55.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:50:55.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:50:55.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33199/jwks.json
03:50:55.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:50:55.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:50:55.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/master-1/data/info.pb
03:50:55.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.122.253
03:50:55.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:50:55.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:50:55.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:50:55.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:50:55.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:50:55.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:50:55.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/master-1/logs
03:50:55.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:50:55.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:50:55.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:50:55.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:50:55.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:50:55.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:50:55.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:50:55.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 02:49:50 UTC on ccycloud.quasar-twsplx.root.comops.site
03:50:55.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4675617
03:50:55.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:50:55.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:55.254261 3645 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:50:55.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:55.255944 3645 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:50:55.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:55.268318 3643 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.122.253" port: 35383 } attempt: 4
03:50:55.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:50:55.272552 3643 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.122.253:35383: Network error: Client connection negotiation failed: client connection to 127.3.122.253:35383: connect: Connection refused (error 111)
03:50:55.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:50:55.278014 3655 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:50:55.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:55.571954 3643 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.122.253" port: 35383 } attempt: 5
03:50:55.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:50:55.577731 3643 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.122.253:35383: Network error: Client connection negotiation failed: client connection to 127.3.122.253:35383: connect: Connection refused (error 111)
03:50:56.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:56.125160 3643 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.122.253" port: 35383 } attempt: 6
03:50:56.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:50:56.134151 3643 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.122.253:35383: Network error: Client connection negotiation failed: client connection to 127.3.122.253:35383: connect: Connection refused (error 111)
03:50:56.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:50:55.280148 3657 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:50:56.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:55.280875 3645 server_base.cc:1034] running on GCE node
03:50:56.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:50:55.278053 3654 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:50:56.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:56.584443 3645 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:50:56.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:56.588150 3645 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:50:56.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:56.589586 3645 hybrid_clock.cc:648] HybridClock initialized: now 1790481056589532 us; error 77 us; skew 500 ppm
03:50:56.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:56.590598 3645 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:50:56.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:56.602083 3645 webserver.cc:460] Webserver started at http://127.3.122.253:44633/ using document root <none> and password file <none>
03:50:56.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:56.603255 3645 fs_manager.cc:359] Metadata directory not provided
03:50:56.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:56.603466 3645 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:50:56.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:56.603881 3645 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:50:56.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:56.609270 3645 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/master-1/data/instance:
03:50:56.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b85e3e306057410f90ced603872e84f4"
03:50:56.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 03:50:56 on dist-test-slave-c26w"
03:50:56.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:56.610599 3645 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/master-1/wal/instance:
03:50:56.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b85e3e306057410f90ced603872e84f4"
03:50:56.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 03:50:56 on dist-test-slave-c26w"
03:50:56.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:56.618736 3645 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.002s sys 0.005s
03:50:56.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:56.625334 3666 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:50:56.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:56.626467 3645 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.004s
03:50:56.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:56.626863 3645 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/master-1/data,/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/master-1/wal
03:50:56.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b85e3e306057410f90ced603872e84f4"
03:50:56.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 03:50:56 on dist-test-slave-c26w"
03:50:56.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:56.627337 3645 fs_report.cc:383] FS layout report
03:50:56.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:50:56.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/master-1/wal
03:50:56.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/master-1/wal
03:50:56.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/master-1/data/data
03:50:56.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:50:56.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:50:56.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:50:56.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:50:56.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:50:56.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:50:56.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:50:56.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:50:56.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:50:56.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:50:56.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:50:56.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:56.677459 3645 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:50:56.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:56.679091 3645 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:50:56.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:56.679596 3645 kserver.cc:163] Server-wide thread pool size limit: 3276
03:50:56.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:56.757427 3645 rpc_server.cc:307] RPC server started. Bound to: 127.3.122.253:35383
03:50:56.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:56.757477 3717 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.3.122.253:35383 every 8 connection(s)
03:50:56.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:56.760222 3645 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/master-1/data/info.pb
03:50:56.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:56.765770 3718 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:50:56.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:56.767946 3563 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskaXaVUt/build/tsan/bin/kudu as pid 3645
03:50:56.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:56.768271 3563 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/master-1/wal/instance
03:50:56.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:56.770251 3563 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskaXaVUt/build/tsan/bin/kudu
03:50:56.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskaXaVUt/build/tsan/bin/kudu
03:50:56.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/master-2/wal
03:50:56.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/master-2/data
03:50:56.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:50:56.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:50:56.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:50:56.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:50:56.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:50:56.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:50:56.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/master-2/logs
03:50:56.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/master-2/data/info.pb
03:50:56.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:50:56.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:50:56.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:50:56.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:50:56.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:50:56.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:50:56.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:50:56.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:50:56.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:50:56.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:50:56.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:50:56.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:50:56.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.122.252:34035
03:50:56.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.122.252
03:50:56.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:50:56.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.122.212:36727
03:50:56.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:50:56.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:50:56.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:50:56.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:50:56.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.122.254:42965,127.3.122.253:35383,127.3.122.252:34035
03:50:56.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:50:56.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33199/jwks.json with env {}
03:50:56.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:56.779793 3718 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.122.254" port: 42965 } has no permanent_uuid. Determining permanent_uuid...
03:50:56.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:56.805728 3718 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.122.253" port: 35383 } has no permanent_uuid. Determining permanent_uuid...
03:50:56.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:56.823282 3718 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.122.252" port: 34035 } has no permanent_uuid. Determining permanent_uuid...
03:50:56.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:50:56.825805 3667 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.122.252:34035: connect: Connection refused (error 111)
03:50:56.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:50:56.831893 3718 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.122.252:34035: Network error: Client connection negotiation failed: client connection to 127.3.122.252:34035: connect: Connection refused (error 111)
03:50:56.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:56.881459 3718 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.122.252" port: 34035 } attempt: 1
03:50:56.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:50:56.885942 3718 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.122.252:34035: Network error: Client connection negotiation failed: client connection to 127.3.122.252:34035: connect: Connection refused (error 111)
03:50:56.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:56.954435 3718 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.122.252" port: 34035 } attempt: 2
03:50:56.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:50:56.958544 3718 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.122.252:34035: Network error: Client connection negotiation failed: client connection to 127.3.122.252:34035: connect: Connection refused (error 111)
03:50:57.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:57.050042 3718 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.122.252" port: 34035 } attempt: 3
03:50:57.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:50:57.054391 3718 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.122.252:34035: Network error: Client connection negotiation failed: client connection to 127.3.122.252:34035: connect: Connection refused (error 111)
03:50:57.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:50:57.107791 3720 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:50:57.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:50:57.108347 3720 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:50:57.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:50:57.108800 3720 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:50:57.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:50:57.139576 3720 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:50:57.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:50:57.139869 3720 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:50:57.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:50:57.140056 3720 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:50:57.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:50:57.140267 3720 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:50:57.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:50:57.140455 3720 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:50:57.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:50:57.140602 3720 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33199/jwks.json
03:50:57.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:57.180889 3720 master_runner.cc:386] Master server non-default flags:
03:50:57.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:50:57.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.122.212:36727
03:50:57.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:50:57.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:50:57.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/master-2/data
03:50:57.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/master-2/wal
03:50:57.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:50:57.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.122.254:42965,127.3.122.253:35383,127.3.122.252:34035
03:50:57.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:50:57.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:50:57.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:50:57.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.122.252:34035
03:50:57.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:50:57.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:50:57.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:50:57.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33199/jwks.json
03:50:57.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:50:57.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:50:57.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/master-2/data/info.pb
03:50:57.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.122.252
03:50:57.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:50:57.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:50:57.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:50:57.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:50:57.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:50:57.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:50:57.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/master-2/logs
03:50:57.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:50:57.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:50:57.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:50:57.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:50:57.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:50:57.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:50:57.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:50:57.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 02:49:50 UTC on ccycloud.quasar-twsplx.root.comops.site
03:50:57.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4675617
03:50:57.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:50:57.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:57.182319 3720 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:50:57.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:57.184532 3720 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:50:57.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:57.194598 3643 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.122.253" port: 35383 } attempt: 7
03:50:57.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:57.197781 3718 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.122.252" port: 34035 } attempt: 4
03:50:57.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:50:57.203759 3718 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.122.252:34035: Network error: Client connection negotiation failed: client connection to 127.3.122.252:34035: connect: Connection refused (error 111)
03:50:57.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:50:57.211421 3730 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:50:57.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:57.220108 3643 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.122.252" port: 34035 } has no permanent_uuid. Determining permanent_uuid...
03:50:57.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:50:57.234298 3643 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.122.252:34035: Network error: Client connection negotiation failed: client connection to 127.3.122.252:34035: connect: Connection refused (error 111)
03:50:57.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:57.293378 3643 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.122.252" port: 34035 } attempt: 1
03:50:57.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:50:57.297765 3643 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.122.252:34035: Network error: Client connection negotiation failed: client connection to 127.3.122.252:34035: connect: Connection refused (error 111)
03:50:57.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:57.379197 3643 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.122.252" port: 34035 } attempt: 2
03:50:57.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:50:57.383975 3643 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.122.252:34035: Network error: Client connection negotiation failed: client connection to 127.3.122.252:34035: connect: Connection refused (error 111)
03:50:57.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:57.469401 3643 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.122.252" port: 34035 } attempt: 3
03:50:57.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:50:57.474130 3643 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.122.252:34035: Network error: Client connection negotiation failed: client connection to 127.3.122.252:34035: connect: Connection refused (error 111)
03:50:57.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:57.503242 3718 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.122.252" port: 34035 } attempt: 5
03:50:57.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:50:57.508316 3718 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.122.252:34035: Network error: Client connection negotiation failed: client connection to 127.3.122.252:34035: connect: Connection refused (error 111)
03:50:57.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:57.614609 3643 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.122.252" port: 34035 } attempt: 4
03:50:57.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:50:57.619292 3643 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.122.252:34035: Network error: Client connection negotiation failed: client connection to 127.3.122.252:34035: connect: Connection refused (error 111)
03:50:57.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:57.902794 3643 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.122.252" port: 34035 } attempt: 5
03:50:57.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:50:57.907807 3643 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.122.252:34035: Network error: Client connection negotiation failed: client connection to 127.3.122.252:34035: connect: Connection refused (error 111)
03:50:58.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:58.055739 3718 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.122.252" port: 34035 } attempt: 6
03:50:58.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:50:58.063946 3718 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.122.252:34035: Network error: Client connection negotiation failed: client connection to 127.3.122.252:34035: connect: Connection refused (error 111)
03:50:58.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:50:57.213932 3732 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:50:58.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:50:57.232259 3734 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:50:58.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:57.230899 3720 server_base.cc:1034] running on GCE node
03:50:58.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:58.442299 3720 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:50:58.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:58.445534 3720 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:50:58.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:58.446993 3720 hybrid_clock.cc:648] HybridClock initialized: now 1790481058446951 us; error 64 us; skew 500 ppm
03:50:58.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:58.447788 3720 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:50:58.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:58.454628 3720 webserver.cc:460] Webserver started at http://127.3.122.252:33621/ using document root <none> and password file <none>
03:50:58.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:58.455590 3720 fs_manager.cc:359] Metadata directory not provided
03:50:58.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:58.455775 3720 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:50:58.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:58.456142 3720 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:50:58.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:58.460232 3643 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.122.252" port: 34035 } attempt: 6
03:50:58.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:58.461581 3720 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/master-2/data/instance:
03:50:58.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fd438d117f484473abcfcc892d6b36d5"
03:50:58.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 03:50:58 on dist-test-slave-c26w"
03:50:58.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:58.463672 3720 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/master-2/wal/instance:
03:50:58.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fd438d117f484473abcfcc892d6b36d5"
03:50:58.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 03:50:58 on dist-test-slave-c26w"
03:50:58.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:50:58.467456 3643 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.122.252:34035: Network error: Client connection negotiation failed: client connection to 127.3.122.252:34035: connect: Connection refused (error 111)
03:50:58.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:58.472802 3720 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.009s sys 0.000s
03:50:58.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:58.479477 3747 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:50:58.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:58.480690 3720 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.001s sys 0.003s
03:50:58.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:58.480985 3720 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/master-2/data,/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/master-2/wal
03:50:58.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fd438d117f484473abcfcc892d6b36d5"
03:50:58.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 03:50:58 on dist-test-slave-c26w"
03:50:58.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:58.481452 3720 fs_report.cc:383] FS layout report
03:50:58.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:50:58.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/master-2/wal
03:50:58.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/master-2/wal
03:50:58.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/master-2/data/data
03:50:58.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:50:58.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:50:58.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:50:58.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:50:58.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:50:58.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:50:58.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:50:58.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:50:58.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:50:58.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:50:58.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:50:58.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:58.516119 3720 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:50:58.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:58.517923 3720 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:50:58.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:58.518337 3720 kserver.cc:163] Server-wide thread pool size limit: 3276
03:50:58.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:58.596601 3720 rpc_server.cc:307] RPC server started. Bound to: 127.3.122.252:34035
03:50:58.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:58.596711 3798 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.3.122.252:34035 every 8 connection(s)
03:50:58.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:58.599236 3720 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/master-2/data/info.pb
03:50:58.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:58.605664 3799 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:50:58.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:58.608696 3563 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskaXaVUt/build/tsan/bin/kudu as pid 3720
03:50:58.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:58.609218 3563 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/master-2/wal/instance
03:50:58.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:58.618645 3799 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.122.254" port: 42965 } has no permanent_uuid. Determining permanent_uuid...
03:50:58.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:58.653909 3799 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.122.253" port: 35383 } has no permanent_uuid. Determining permanent_uuid...
03:50:58.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:58.667471 3799 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.122.252" port: 34035 } has no permanent_uuid. Determining permanent_uuid...
03:50:58.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:58.685145 3799 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P fd438d117f484473abcfcc892d6b36d5: Bootstrap starting.
03:50:58.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:58.691479 3799 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P fd438d117f484473abcfcc892d6b36d5: Neither blocks nor log segments found. Creating new log.
03:50:58.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:58.694062 3799 log.cc:824] T 00000000000000000000000000000000 P fd438d117f484473abcfcc892d6b36d5: Log is configured to *not* fsync() on all Append() calls
03:50:58.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:58.699177 3799 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P fd438d117f484473abcfcc892d6b36d5: No bootstrap required, opened a new log
03:50:58.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:58.720489 3799 raft_consensus.cc:348] T 00000000000000000000000000000000 P fd438d117f484473abcfcc892d6b36d5 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8df192a545f94500a26d100e6cefc726" member_type: VOTER last_known_addr { host: "127.3.122.254" port: 42965 } } peers { permanent_uuid: "b85e3e306057410f90ced603872e84f4" member_type: VOTER last_known_addr { host: "127.3.122.253" port: 35383 } } peers { permanent_uuid: "fd438d117f484473abcfcc892d6b36d5" member_type: VOTER last_known_addr { host: "127.3.122.252" port: 34035 } }
03:50:58.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:58.721333 3799 raft_consensus.cc:374] T 00000000000000000000000000000000 P fd438d117f484473abcfcc892d6b36d5 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:50:58.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:58.721611 3799 raft_consensus.cc:729] T 00000000000000000000000000000000 P fd438d117f484473abcfcc892d6b36d5 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fd438d117f484473abcfcc892d6b36d5, State: Initialized, Role: FOLLOWER
03:50:58.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:58.722509 3799 consensus_queue.cc:260] T 00000000000000000000000000000000 P fd438d117f484473abcfcc892d6b36d5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8df192a545f94500a26d100e6cefc726" member_type: VOTER last_known_addr { host: "127.3.122.254" port: 42965 } } peers { permanent_uuid: "b85e3e306057410f90ced603872e84f4" member_type: VOTER last_known_addr { host: "127.3.122.253" port: 35383 } } peers { permanent_uuid: "fd438d117f484473abcfcc892d6b36d5" member_type: VOTER last_known_addr { host: "127.3.122.252" port: 34035 } }
03:50:58.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:58.725387 3808 sys_catalog.cc:455] T 00000000000000000000000000000000 P fd438d117f484473abcfcc892d6b36d5 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8df192a545f94500a26d100e6cefc726" member_type: VOTER last_known_addr { host: "127.3.122.254" port: 42965 } } peers { permanent_uuid: "b85e3e306057410f90ced603872e84f4" member_type: VOTER last_known_addr { host: "127.3.122.253" port: 35383 } } peers { permanent_uuid: "fd438d117f484473abcfcc892d6b36d5" member_type: VOTER last_known_addr { host: "127.3.122.252" port: 34035 } } }
03:50:58.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:58.725978 3808 sys_catalog.cc:458] T 00000000000000000000000000000000 P fd438d117f484473abcfcc892d6b36d5 [sys.catalog]: This master's current role is: FOLLOWER
03:50:58.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:58.726755 3799 sys_catalog.cc:564] T 00000000000000000000000000000000 P fd438d117f484473abcfcc892d6b36d5 [sys.catalog]: configured and running, proceeding with master startup.
03:50:58.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:50:58.759703 3819 catalog_manager.cc:1558] T 00000000000000000000000000000000 P fd438d117f484473abcfcc892d6b36d5: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:50:58.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:50:58.760083 3819 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:50:58.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:50:58.773746 3753 tablet.cc:2369] T 00000000000000000000000000000000 P fd438d117f484473abcfcc892d6b36d5: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:50:58.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:58.750538 3720 master_runner.cc:186] Error getting master registration for 127.3.122.254:42965: OK, instance_id { permanent_uuid: "8df192a545f94500a26d100e6cefc726" instance_seqno: 1790481054814474 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
03:50:58.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:58.785152 3720 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:50:58.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:58.945744 3808 raft_consensus.cc:482] T 00000000000000000000000000000000 P fd438d117f484473abcfcc892d6b36d5 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:50:58.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:58.946318 3808 raft_consensus.cc:504] T 00000000000000000000000000000000 P fd438d117f484473abcfcc892d6b36d5 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8df192a545f94500a26d100e6cefc726" member_type: VOTER last_known_addr { host: "127.3.122.254" port: 42965 } } peers { permanent_uuid: "b85e3e306057410f90ced603872e84f4" member_type: VOTER last_known_addr { host: "127.3.122.253" port: 35383 } } peers { permanent_uuid: "fd438d117f484473abcfcc892d6b36d5" member_type: VOTER last_known_addr { host: "127.3.122.252" port: 34035 } }
03:50:58.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:58.949499 3808 leader_election.cc:288] T 00000000000000000000000000000000 P fd438d117f484473abcfcc892d6b36d5 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8df192a545f94500a26d100e6cefc726 (127.3.122.254:42965), b85e3e306057410f90ced603872e84f4 (127.3.122.253:35383)
03:50:59.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.124351 3718 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.122.252" port: 34035 } attempt: 7
03:50:59.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.148313 3718 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b85e3e306057410f90ced603872e84f4: Bootstrap starting.
03:50:59.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.156764 3718 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P b85e3e306057410f90ced603872e84f4: Neither blocks nor log segments found. Creating new log.
03:50:59.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.159408 3718 log.cc:824] T 00000000000000000000000000000000 P b85e3e306057410f90ced603872e84f4: Log is configured to *not* fsync() on all Append() calls
03:50:59.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.165797 3718 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b85e3e306057410f90ced603872e84f4: No bootstrap required, opened a new log
03:50:59.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.197081 3718 raft_consensus.cc:348] T 00000000000000000000000000000000 P b85e3e306057410f90ced603872e84f4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8df192a545f94500a26d100e6cefc726" member_type: VOTER last_known_addr { host: "127.3.122.254" port: 42965 } } peers { permanent_uuid: "b85e3e306057410f90ced603872e84f4" member_type: VOTER last_known_addr { host: "127.3.122.253" port: 35383 } } peers { permanent_uuid: "fd438d117f484473abcfcc892d6b36d5" member_type: VOTER last_known_addr { host: "127.3.122.252" port: 34035 } }
03:50:59.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.198786 3718 raft_consensus.cc:374] T 00000000000000000000000000000000 P b85e3e306057410f90ced603872e84f4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:50:59.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.199231 3718 raft_consensus.cc:729] T 00000000000000000000000000000000 P b85e3e306057410f90ced603872e84f4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b85e3e306057410f90ced603872e84f4, State: Initialized, Role: FOLLOWER
03:50:59.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.200467 3718 consensus_queue.cc:260] T 00000000000000000000000000000000 P b85e3e306057410f90ced603872e84f4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8df192a545f94500a26d100e6cefc726" member_type: VOTER last_known_addr { host: "127.3.122.254" port: 42965 } } peers { permanent_uuid: "b85e3e306057410f90ced603872e84f4" member_type: VOTER last_known_addr { host: "127.3.122.253" port: 35383 } } peers { permanent_uuid: "fd438d117f484473abcfcc892d6b36d5" member_type: VOTER last_known_addr { host: "127.3.122.252" port: 34035 } }
03:50:59.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.203891 3823 sys_catalog.cc:455] T 00000000000000000000000000000000 P b85e3e306057410f90ced603872e84f4 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8df192a545f94500a26d100e6cefc726" member_type: VOTER last_known_addr { host: "127.3.122.254" port: 42965 } } peers { permanent_uuid: "b85e3e306057410f90ced603872e84f4" member_type: VOTER last_known_addr { host: "127.3.122.253" port: 35383 } } peers { permanent_uuid: "fd438d117f484473abcfcc892d6b36d5" member_type: VOTER last_known_addr { host: "127.3.122.252" port: 34035 } } }
03:50:59.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.204697 3823 sys_catalog.cc:458] T 00000000000000000000000000000000 P b85e3e306057410f90ced603872e84f4 [sys.catalog]: This master's current role is: FOLLOWER
03:50:59.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.205636 3718 sys_catalog.cc:564] T 00000000000000000000000000000000 P b85e3e306057410f90ced603872e84f4 [sys.catalog]: configured and running, proceeding with master startup.
03:50:59.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.205847 3693 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "fd438d117f484473abcfcc892d6b36d5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b85e3e306057410f90ced603872e84f4" is_pre_election: true
03:50:59.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.206730 3693 raft_consensus.cc:2454] T 00000000000000000000000000000000 P b85e3e306057410f90ced603872e84f4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate fd438d117f484473abcfcc892d6b36d5 in term 0.
03:50:59.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.208792 3748 leader_election.cc:302] T 00000000000000000000000000000000 P fd438d117f484473abcfcc892d6b36d5 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: b85e3e306057410f90ced603872e84f4, fd438d117f484473abcfcc892d6b36d5; no voters:
03:50:59.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.209869 3808 raft_consensus.cc:2781] T 00000000000000000000000000000000 P fd438d117f484473abcfcc892d6b36d5 [term 0 FOLLOWER]: Leader pre-election won for term 1
03:50:59.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.210229 3808 raft_consensus.cc:482] T 00000000000000000000000000000000 P fd438d117f484473abcfcc892d6b36d5 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
03:50:59.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.210539 3808 raft_consensus.cc:3037] T 00000000000000000000000000000000 P fd438d117f484473abcfcc892d6b36d5 [term 0 FOLLOWER]: Advancing to term 1
03:50:59.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.215301 3808 raft_consensus.cc:504] T 00000000000000000000000000000000 P fd438d117f484473abcfcc892d6b36d5 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8df192a545f94500a26d100e6cefc726" member_type: VOTER last_known_addr { host: "127.3.122.254" port: 42965 } } peers { permanent_uuid: "b85e3e306057410f90ced603872e84f4" member_type: VOTER last_known_addr { host: "127.3.122.253" port: 35383 } } peers { permanent_uuid: "fd438d117f484473abcfcc892d6b36d5" member_type: VOTER last_known_addr { host: "127.3.122.252" port: 34035 } }
03:50:59.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.220657 3693 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "fd438d117f484473abcfcc892d6b36d5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b85e3e306057410f90ced603872e84f4"
03:50:59.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.221405 3693 raft_consensus.cc:3037] T 00000000000000000000000000000000 P b85e3e306057410f90ced603872e84f4 [term 0 FOLLOWER]: Advancing to term 1
03:50:59.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.229573 3808 leader_election.cc:288] T 00000000000000000000000000000000 P fd438d117f484473abcfcc892d6b36d5 [CANDIDATE]: Term 1 election: Requested vote from peers 8df192a545f94500a26d100e6cefc726 (127.3.122.254:42965), b85e3e306057410f90ced603872e84f4 (127.3.122.253:35383)
03:50:59.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.232566 3693 raft_consensus.cc:2454] T 00000000000000000000000000000000 P b85e3e306057410f90ced603872e84f4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate fd438d117f484473abcfcc892d6b36d5 in term 1.
03:50:59.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.234128 3748 leader_election.cc:302] T 00000000000000000000000000000000 P fd438d117f484473abcfcc892d6b36d5 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: b85e3e306057410f90ced603872e84f4, fd438d117f484473abcfcc892d6b36d5; no voters:
03:50:59.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.235162 3808 raft_consensus.cc:2781] T 00000000000000000000000000000000 P fd438d117f484473abcfcc892d6b36d5 [term 1 FOLLOWER]: Leader election won for term 1
03:50:59.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.238899 3808 raft_consensus.cc:686] T 00000000000000000000000000000000 P fd438d117f484473abcfcc892d6b36d5 [term 1 LEADER]: Becoming Leader. State: Replica: fd438d117f484473abcfcc892d6b36d5, State: Running, Role: LEADER
03:50:59.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.239866 3808 consensus_queue.cc:237] T 00000000000000000000000000000000 P fd438d117f484473abcfcc892d6b36d5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8df192a545f94500a26d100e6cefc726" member_type: VOTER last_known_addr { host: "127.3.122.254" port: 42965 } } peers { permanent_uuid: "b85e3e306057410f90ced603872e84f4" member_type: VOTER last_known_addr { host: "127.3.122.253" port: 35383 } } peers { permanent_uuid: "fd438d117f484473abcfcc892d6b36d5" member_type: VOTER last_known_addr { host: "127.3.122.252" port: 34035 } }
03:50:59.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.254212 3830 sys_catalog.cc:455] T 00000000000000000000000000000000 P fd438d117f484473abcfcc892d6b36d5 [sys.catalog]: SysCatalogTable state changed. Reason: New leader fd438d117f484473abcfcc892d6b36d5. Latest consensus state: current_term: 1 leader_uuid: "fd438d117f484473abcfcc892d6b36d5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8df192a545f94500a26d100e6cefc726" member_type: VOTER last_known_addr { host: "127.3.122.254" port: 42965 } } peers { permanent_uuid: "b85e3e306057410f90ced603872e84f4" member_type: VOTER last_known_addr { host: "127.3.122.253" port: 35383 } } peers { permanent_uuid: "fd438d117f484473abcfcc892d6b36d5" member_type: VOTER last_known_addr { host: "127.3.122.252" port: 34035 } } }
03:50:59.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.255062 3830 sys_catalog.cc:458] T 00000000000000000000000000000000 P fd438d117f484473abcfcc892d6b36d5 [sys.catalog]: This master's current role is: LEADER
03:50:59.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.266928 3836 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
03:50:59.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.274516 3836 catalog_manager.cc:1484] Initializing Kudu cluster ID...
03:50:59.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:50:59.289139 3837 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b85e3e306057410f90ced603872e84f4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:50:59.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:50:59.289847 3837 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:50:59.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.299686 3693 raft_consensus.cc:1261] T 00000000000000000000000000000000 P b85e3e306057410f90ced603872e84f4 [term 1 FOLLOWER]: Refusing update from remote peer fd438d117f484473abcfcc892d6b36d5: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
03:50:59.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.301841 3808 consensus_queue.cc:1035] T 00000000000000000000000000000000 P fd438d117f484473abcfcc892d6b36d5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b85e3e306057410f90ced603872e84f4" member_type: VOTER last_known_addr { host: "127.3.122.253" port: 35383 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:50:59.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.275518 3645 master_runner.cc:186] Error getting master registration for 127.3.122.254:42965: OK, instance_id { permanent_uuid: "8df192a545f94500a26d100e6cefc726" instance_seqno: 1790481054814474 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
03:50:59.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.336319 3645 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:50:59.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.367996 3823 sys_catalog.cc:455] T 00000000000000000000000000000000 P b85e3e306057410f90ced603872e84f4 [sys.catalog]: SysCatalogTable state changed. Reason: New leader fd438d117f484473abcfcc892d6b36d5. Latest consensus state: current_term: 1 leader_uuid: "fd438d117f484473abcfcc892d6b36d5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8df192a545f94500a26d100e6cefc726" member_type: VOTER last_known_addr { host: "127.3.122.254" port: 42965 } } peers { permanent_uuid: "b85e3e306057410f90ced603872e84f4" member_type: VOTER last_known_addr { host: "127.3.122.253" port: 35383 } } peers { permanent_uuid: "fd438d117f484473abcfcc892d6b36d5" member_type: VOTER last_known_addr { host: "127.3.122.252" port: 34035 } } }
03:50:59.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.374155 3823 sys_catalog.cc:458] T 00000000000000000000000000000000 P b85e3e306057410f90ced603872e84f4 [sys.catalog]: This master's current role is: FOLLOWER
03:50:59.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.386358 3830 sys_catalog.cc:455] T 00000000000000000000000000000000 P fd438d117f484473abcfcc892d6b36d5 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "fd438d117f484473abcfcc892d6b36d5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8df192a545f94500a26d100e6cefc726" member_type: VOTER last_known_addr { host: "127.3.122.254" port: 42965 } } peers { permanent_uuid: "b85e3e306057410f90ced603872e84f4" member_type: VOTER last_known_addr { host: "127.3.122.253" port: 35383 } } peers { permanent_uuid: "fd438d117f484473abcfcc892d6b36d5" member_type: VOTER last_known_addr { host: "127.3.122.252" port: 34035 } } }
03:50:59.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.387542 3830 sys_catalog.cc:458] T 00000000000000000000000000000000 P fd438d117f484473abcfcc892d6b36d5 [sys.catalog]: This master's current role is: LEADER
03:50:59.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.392239 3823 sys_catalog.cc:455] T 00000000000000000000000000000000 P b85e3e306057410f90ced603872e84f4 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "fd438d117f484473abcfcc892d6b36d5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8df192a545f94500a26d100e6cefc726" member_type: VOTER last_known_addr { host: "127.3.122.254" port: 42965 } } peers { permanent_uuid: "b85e3e306057410f90ced603872e84f4" member_type: VOTER last_known_addr { host: "127.3.122.253" port: 35383 } } peers { permanent_uuid: "fd438d117f484473abcfcc892d6b36d5" member_type: VOTER last_known_addr { host: "127.3.122.252" port: 34035 } } }
03:50:59.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.392939 3823 sys_catalog.cc:458] T 00000000000000000000000000000000 P b85e3e306057410f90ced603872e84f4 [sys.catalog]: This master's current role is: FOLLOWER
03:50:59.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.400024 3836 catalog_manager.cc:1347] Generated new cluster ID: f20cc00273cc42f7920e411a38925e8c
03:50:59.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.400324 3836 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
03:50:59.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.483681 3836 catalog_manager.cc:1370] Generated new certificate authority record
03:50:59.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.489414 3836 catalog_manager.cc:1504] Loading token signing keys...
03:50:59.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.500844 3643 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.122.252" port: 34035 } attempt: 7
03:50:59.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.526237 3836 catalog_manager.cc:5868] T 00000000000000000000000000000000 P fd438d117f484473abcfcc892d6b36d5: Generated new TSK 0
03:50:59.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.530602 3836 catalog_manager.cc:1514] Initializing in-progress tserver states...
03:50:59.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.548996 3563 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskaXaVUt/build/tsan/bin/kudu
03:50:59.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskaXaVUt/build/tsan/bin/kudu
03:50:59.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/ts-0/wal
03:50:59.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/ts-0/data
03:50:59.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:50:59.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:50:59.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:50:59.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:50:59.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:50:59.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:50:59.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/ts-0/logs
03:50:59.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/ts-0/data/info.pb
03:50:59.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:50:59.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:50:59.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:50:59.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:50:59.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:50:59.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:50:59.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:50:59.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:50:59.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:50:59.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:50:59.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.122.193:0
03:50:59.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.122.193
03:50:59.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.122.193
03:50:59.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:50:59.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.122.254:42965,127.3.122.253:35383,127.3.122.252:34035
03:50:59.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.122.212:36727
03:50:59.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:50:59.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:50:59.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
03:50:59.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.565521 3643 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8df192a545f94500a26d100e6cefc726: Bootstrap starting.
03:50:59.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.573493 3643 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 8df192a545f94500a26d100e6cefc726: Neither blocks nor log segments found. Creating new log.
03:50:59.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.576550 3643 log.cc:824] T 00000000000000000000000000000000 P 8df192a545f94500a26d100e6cefc726: Log is configured to *not* fsync() on all Append() calls
03:50:59.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.588743 3643 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8df192a545f94500a26d100e6cefc726: No bootstrap required, opened a new log
03:50:59.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.623274 3643 raft_consensus.cc:348] T 00000000000000000000000000000000 P 8df192a545f94500a26d100e6cefc726 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8df192a545f94500a26d100e6cefc726" member_type: VOTER last_known_addr { host: "127.3.122.254" port: 42965 } } peers { permanent_uuid: "b85e3e306057410f90ced603872e84f4" member_type: VOTER last_known_addr { host: "127.3.122.253" port: 35383 } } peers { permanent_uuid: "fd438d117f484473abcfcc892d6b36d5" member_type: VOTER last_known_addr { host: "127.3.122.252" port: 34035 } }
03:50:59.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.624460 3643 raft_consensus.cc:374] T 00000000000000000000000000000000 P 8df192a545f94500a26d100e6cefc726 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:50:59.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.624944 3643 raft_consensus.cc:729] T 00000000000000000000000000000000 P 8df192a545f94500a26d100e6cefc726 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8df192a545f94500a26d100e6cefc726, State: Initialized, Role: FOLLOWER
03:50:59.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.626099 3643 consensus_queue.cc:260] T 00000000000000000000000000000000 P 8df192a545f94500a26d100e6cefc726 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8df192a545f94500a26d100e6cefc726" member_type: VOTER last_known_addr { host: "127.3.122.254" port: 42965 } } peers { permanent_uuid: "b85e3e306057410f90ced603872e84f4" member_type: VOTER last_known_addr { host: "127.3.122.253" port: 35383 } } peers { permanent_uuid: "fd438d117f484473abcfcc892d6b36d5" member_type: VOTER last_known_addr { host: "127.3.122.252" port: 34035 } }
03:50:59.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.639940 3643 sys_catalog.cc:564] T 00000000000000000000000000000000 P 8df192a545f94500a26d100e6cefc726 [sys.catalog]: configured and running, proceeding with master startup.
03:50:59.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.640537 3616 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 8df192a545f94500a26d100e6cefc726 [term 0 FOLLOWER]: Advancing to term 1
03:50:59.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.641228 3847 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8df192a545f94500a26d100e6cefc726 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8df192a545f94500a26d100e6cefc726" member_type: VOTER last_known_addr { host: "127.3.122.254" port: 42965 } } peers { permanent_uuid: "b85e3e306057410f90ced603872e84f4" member_type: VOTER last_known_addr { host: "127.3.122.253" port: 35383 } } peers { permanent_uuid: "fd438d117f484473abcfcc892d6b36d5" member_type: VOTER last_known_addr { host: "127.3.122.252" port: 34035 } } }
03:50:59.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.642050 3847 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8df192a545f94500a26d100e6cefc726 [sys.catalog]: This master's current role is: FOLLOWER
03:50:59.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.647133 3617 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "fd438d117f484473abcfcc892d6b36d5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8df192a545f94500a26d100e6cefc726"
03:50:59.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.649068 3618 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "fd438d117f484473abcfcc892d6b36d5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8df192a545f94500a26d100e6cefc726" is_pre_election: true
03:50:59.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.654050 3616 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 8df192a545f94500a26d100e6cefc726 [term 1 FOLLOWER]: Refusing update from remote peer fd438d117f484473abcfcc892d6b36d5: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
03:50:59.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.655820 3808 consensus_queue.cc:1035] T 00000000000000000000000000000000 P fd438d117f484473abcfcc892d6b36d5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8df192a545f94500a26d100e6cefc726" member_type: VOTER last_known_addr { host: "127.3.122.254" port: 42965 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:50:59.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.733333 3847 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8df192a545f94500a26d100e6cefc726 [sys.catalog]: SysCatalogTable state changed. Reason: New leader fd438d117f484473abcfcc892d6b36d5. Latest consensus state: current_term: 1 leader_uuid: "fd438d117f484473abcfcc892d6b36d5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8df192a545f94500a26d100e6cefc726" member_type: VOTER last_known_addr { host: "127.3.122.254" port: 42965 } } peers { permanent_uuid: "b85e3e306057410f90ced603872e84f4" member_type: VOTER last_known_addr { host: "127.3.122.253" port: 35383 } } peers { permanent_uuid: "fd438d117f484473abcfcc892d6b36d5" member_type: VOTER last_known_addr { host: "127.3.122.252" port: 34035 } } }
03:50:59.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.734064 3847 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8df192a545f94500a26d100e6cefc726 [sys.catalog]: This master's current role is: FOLLOWER
03:50:59.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.735296 3855 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8df192a545f94500a26d100e6cefc726 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "fd438d117f484473abcfcc892d6b36d5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8df192a545f94500a26d100e6cefc726" member_type: VOTER last_known_addr { host: "127.3.122.254" port: 42965 } } peers { permanent_uuid: "b85e3e306057410f90ced603872e84f4" member_type: VOTER last_known_addr { host: "127.3.122.253" port: 35383 } } peers { permanent_uuid: "fd438d117f484473abcfcc892d6b36d5" member_type: VOTER last_known_addr { host: "127.3.122.252" port: 34035 } } }
03:50:59.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.736361 3855 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8df192a545f94500a26d100e6cefc726 [sys.catalog]: This master's current role is: FOLLOWER
03:50:59.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.871861 3863 catalog_manager.cc:1259] Loaded cluster ID: f20cc00273cc42f7920e411a38925e8c
03:50:59.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.872180 3863 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 8df192a545f94500a26d100e6cefc726: loading cluster ID for follower catalog manager: success
03:50:59.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.903311 3863 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 8df192a545f94500a26d100e6cefc726: acquiring CA information for follower catalog manager: success
03:50:59.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:50:59.931389 3863 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 8df192a545f94500a26d100e6cefc726: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:51:00.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:00.129802 3865 sys_catalog.cc:455] T 00000000000000000000000000000000 P fd438d117f484473abcfcc892d6b36d5 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "fd438d117f484473abcfcc892d6b36d5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8df192a545f94500a26d100e6cefc726" member_type: VOTER last_known_addr { host: "127.3.122.254" port: 42965 } } peers { permanent_uuid: "b85e3e306057410f90ced603872e84f4" member_type: VOTER last_known_addr { host: "127.3.122.253" port: 35383 } } peers { permanent_uuid: "fd438d117f484473abcfcc892d6b36d5" member_type: VOTER last_known_addr { host: "127.3.122.252" port: 34035 } } }
03:51:00.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:00.130550 3865 sys_catalog.cc:458] T 00000000000000000000000000000000 P fd438d117f484473abcfcc892d6b36d5 [sys.catalog]: This master's current role is: LEADER
03:51:00.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:00.240850 3845 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:51:00.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:00.241907 3845 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:51:00.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:00.242960 3845 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:51:00.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:00.296083 3837 catalog_manager.cc:1259] Loaded cluster ID: f20cc00273cc42f7920e411a38925e8c
03:51:00.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:00.296756 3837 catalog_manager.cc:1552] T 00000000000000000000000000000000 P b85e3e306057410f90ced603872e84f4: loading cluster ID for follower catalog manager: success
03:51:00.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:00.302531 3845 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:51:00.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:00.304023 3845 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.3.122.193
03:51:00.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:00.310956 3837 catalog_manager.cc:1574] T 00000000000000000000000000000000 P b85e3e306057410f90ced603872e84f4: acquiring CA information for follower catalog manager: success
03:51:00.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:00.329104 3837 catalog_manager.cc:1602] T 00000000000000000000000000000000 P b85e3e306057410f90ced603872e84f4: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:51:00.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:00.367115 3845 tablet_server_runner.cc:78] Tablet server non-default flags:
03:51:00.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:51:00.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.122.212:36727
03:51:00.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:51:00.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:51:00.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/ts-0/data
03:51:00.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/ts-0/wal
03:51:00.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:51:00.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:51:00.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.122.193:0
03:51:00.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:51:00.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:51:00.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:51:00.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/ts-0/data/info.pb
03:51:00.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.122.193
03:51:00.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:51:00.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.122.254:42965,127.3.122.253:35383,127.3.122.252:34035
03:51:00.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:51:00.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:51:00.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:51:00.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:51:00.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:51:00.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.122.193
03:51:00.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/ts-0/logs
03:51:00.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:51:00.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:51:00.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:51:00.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:51:00.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:51:00.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:51:00.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:51:00.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 02:49:50 UTC on ccycloud.quasar-twsplx.root.comops.site
03:51:00.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4675617
03:51:00.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:51:00.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:00.368829 3845 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:51:00.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:00.370962 3845 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:51:00.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:00.420080 3873 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:51:02.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:02.409317 3714 debug-util.cc:398] Leaking SignalData structure 0x7b080007b440 after lost signal to thread 3649
03:51:02.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:02.410145 3714 debug-util.cc:398] Leaking SignalData structure 0x7b08000b7e80 after lost signal to thread 3717
03:51:02.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:00.422627 3872 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:51:02.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:01.794000 3871 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 3845
03:51:02.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:02.737916 3874 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2319 milliseconds
03:51:02.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:02.740085 3875 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:51:02.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:02.738188 3845 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 2.325s user 0.298s sys 0.639s
03:51:02.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:02.740731 3845 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 2.328s user 0.298s sys 0.639s
03:51:02.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:02.741268 3845 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:51:02.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:02.744979 3845 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:51:02.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:02.747900 3845 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:51:02.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:02.749495 3845 hybrid_clock.cc:648] HybridClock initialized: now 1790481062749406 us; error 49 us; skew 500 ppm
03:51:02.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:02.750434 3845 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:51:02.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:02.767409 3845 webserver.cc:460] Webserver started at http://127.3.122.193:44319/ using document root <none> and password file <none>
03:51:02.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:02.768460 3845 fs_manager.cc:359] Metadata directory not provided
03:51:02.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:02.768728 3845 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:51:02.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:02.769236 3845 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:51:02.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:02.774142 3845 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/ts-0/data/instance:
03:51:02.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "991b1cbb02dc4ba7b0bb8a43c9b139c1"
03:51:02.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 03:51:02 on dist-test-slave-c26w"
03:51:02.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:02.775523 3845 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/ts-0/wal/instance:
03:51:02.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "991b1cbb02dc4ba7b0bb8a43c9b139c1"
03:51:02.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 03:51:02 on dist-test-slave-c26w"
03:51:02.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:02.784423 3845 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.004s sys 0.004s
03:51:02.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:02.792114 3883 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:51:02.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:02.793398 3845 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.001s sys 0.003s
03:51:02.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:02.793762 3845 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/ts-0/data,/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/ts-0/wal
03:51:02.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "991b1cbb02dc4ba7b0bb8a43c9b139c1"
03:51:02.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 03:51:02 on dist-test-slave-c26w"
03:51:02.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:02.794147 3845 fs_report.cc:383] FS layout report
03:51:02.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:51:02.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/ts-0/wal
03:51:02.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/ts-0/wal
03:51:02.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/ts-0/data/data
03:51:02.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:51:02.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:51:02.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:51:02.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:51:02.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:51:02.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:51:02.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:51:02.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:51:02.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:51:02.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:51:02.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:51:02.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:02.834556 3845 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:51:02.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:02.836509 3845 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:51:02.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:02.837515 3845 kserver.cc:163] Server-wide thread pool size limit: 3276
03:51:02.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:02.859403 3845 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
03:51:02.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:02.873533 3845 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:51:02.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:02.873859 3845 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
03:51:02.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:02.874142 3845 ts_tablet_manager.cc:610] Registered 0 tablets
03:51:02.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:02.874372 3845 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
03:51:03.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:03.152236 3845 rpc_server.cc:307] RPC server started. Bound to: 127.3.122.193:37293
03:51:03.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:03.152354 3996 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.3.122.193:37293 every 8 connection(s)
03:51:03.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:03.155236 3845 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/ts-0/data/info.pb
03:51:03.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:03.159016 3563 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskaXaVUt/build/tsan/bin/kudu as pid 3845
03:51:03.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:03.159698 3563 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/ts-0/wal/instance
03:51:03.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:03.188171 3563 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskaXaVUt/build/tsan/bin/kudu
03:51:03.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskaXaVUt/build/tsan/bin/kudu
03:51:03.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/ts-1/wal
03:51:03.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/ts-1/data
03:51:03.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:51:03.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:51:03.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:51:03.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:51:03.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:51:03.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:51:03.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/ts-1/logs
03:51:03.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/ts-1/data/info.pb
03:51:03.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:51:03.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:51:03.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:51:03.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:51:03.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:51:03.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:51:03.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:51:03.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:51:03.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:51:03.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:51:03.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.122.194:0
03:51:03.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.122.194
03:51:03.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.122.194
03:51:03.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:51:03.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.122.254:42965,127.3.122.253:35383,127.3.122.252:34035
03:51:03.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.122.212:36727
03:51:03.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:51:03.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:51:03.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
03:51:03.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:03.229809 3997 heartbeater.cc:347] Connected to a master server at 127.3.122.252:34035
03:51:03.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:03.230309 3997 heartbeater.cc:464] Registering TS with master...
03:51:03.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:03.231535 3997 heartbeater.cc:511] Master 127.3.122.252:34035 requested a full tablet report, sending...
03:51:03.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:03.234556 3764 ts_manager.cc:194] Registered new tserver with Master: 991b1cbb02dc4ba7b0bb8a43c9b139c1 (127.3.122.193:37293)
03:51:03.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:03.237967 3764 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.3.122.193:38349
03:51:03.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:03.246754 3998 heartbeater.cc:347] Connected to a master server at 127.3.122.253:35383
03:51:03.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:03.247191 3998 heartbeater.cc:464] Registering TS with master...
03:51:03.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:03.247958 3998 heartbeater.cc:511] Master 127.3.122.253:35383 requested a full tablet report, sending...
03:51:03.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:03.251145 3683 ts_manager.cc:194] Registered new tserver with Master: 991b1cbb02dc4ba7b0bb8a43c9b139c1 (127.3.122.193:37293)
03:51:03.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:03.254115 4001 heartbeater.cc:347] Connected to a master server at 127.3.122.254:42965
03:51:03.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:03.254513 4001 heartbeater.cc:464] Registering TS with master...
03:51:03.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:03.255280 4001 heartbeater.cc:511] Master 127.3.122.254:42965 requested a full tablet report, sending...
03:51:03.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:03.258360 3607 ts_manager.cc:194] Registered new tserver with Master: 991b1cbb02dc4ba7b0bb8a43c9b139c1 (127.3.122.193:37293)
03:51:03.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:03.630082 4004 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:51:03.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:03.630662 4004 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:51:03.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:03.631356 4004 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:51:03.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:03.668234 4004 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:51:03.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:03.669240 4004 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.3.122.194
03:51:03.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:03.705749 4004 tablet_server_runner.cc:78] Tablet server non-default flags:
03:51:03.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:51:03.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.122.212:36727
03:51:03.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:51:03.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:51:03.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/ts-1/data
03:51:03.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/ts-1/wal
03:51:03.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:51:03.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:51:03.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.122.194:0
03:51:03.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:51:03.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:51:03.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:51:03.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/ts-1/data/info.pb
03:51:03.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.122.194
03:51:03.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:51:03.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.122.254:42965,127.3.122.253:35383,127.3.122.252:34035
03:51:03.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:51:03.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:51:03.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:51:03.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:51:03.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:51:03.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.122.194
03:51:03.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/ts-1/logs
03:51:03.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:51:03.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:51:03.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:51:03.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:51:03.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:51:03.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:51:03.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:51:03.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 02:49:50 UTC on ccycloud.quasar-twsplx.root.comops.site
03:51:03.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4675617
03:51:03.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:51:03.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:03.707341 4004 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:51:03.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:03.709440 4004 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:51:03.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:03.753881 4015 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:51:04.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:04.243151 3997 heartbeater.cc:503] Master 127.3.122.252:34035 was elected leader, sending a full tablet report...
03:51:04.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:04.569175 3795 debug-util.cc:398] Leaking SignalData structure 0x7b080003f860 after lost signal to thread 3725
03:51:04.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:04.570246 3795 debug-util.cc:398] Leaking SignalData structure 0x7b08000a2080 after lost signal to thread 3798
03:51:05.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:05.138933 4013 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 4004
03:51:05.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:05.748337 4013 kernel_stack_watchdog.cc:198] Thread 4004 stuck at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:637 for 378ms:
03:51:05.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
03:51:05.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
03:51:05.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
03:51:05.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
03:51:05.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:05.748623 4004 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 1.991s user 0.395s sys 0.725s
03:51:05.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:05.749388 4004 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 1.992s user 0.395s sys 0.725s
03:51:05.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:03.755573 4014 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:51:05.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:05.751704 4017 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:51:05.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:05.756071 4016 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1993 milliseconds
03:51:05.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:05.757282 4004 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:51:05.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:05.760356 4004 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:51:05.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:05.764505 4004 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:51:05.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:05.766044 4004 hybrid_clock.cc:648] HybridClock initialized: now 1790481065765965 us; error 100 us; skew 500 ppm
03:51:05.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:05.767241 4004 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:51:05.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:05.776715 4004 webserver.cc:460] Webserver started at http://127.3.122.194:38553/ using document root <none> and password file <none>
03:51:05.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:05.778111 4004 fs_manager.cc:359] Metadata directory not provided
03:51:05.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:05.778389 4004 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:51:05.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:05.778972 4004 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:51:05.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:05.787060 4004 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/ts-1/data/instance:
03:51:05.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ccfd9c961c50436dade821d75b3d9433"
03:51:05.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 03:51:05 on dist-test-slave-c26w"
03:51:05.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:05.788828 4004 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/ts-1/wal/instance:
03:51:05.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ccfd9c961c50436dade821d75b3d9433"
03:51:05.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 03:51:05 on dist-test-slave-c26w"
03:51:05.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:05.800117 4004 fs_manager.cc:691] Time spent creating directory manager: real 0.010s user 0.006s sys 0.006s
03:51:05.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:05.808939 4025 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:51:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:05.810783 4004 fs_manager.cc:725] Time spent opening block manager: real 0.006s user 0.004s sys 0.001s
03:51:05.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:05.811434 4004 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/ts-1/data,/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/ts-1/wal
03:51:05.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ccfd9c961c50436dade821d75b3d9433"
03:51:05.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 03:51:05 on dist-test-slave-c26w"
03:51:05.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:05.812049 4004 fs_report.cc:383] FS layout report
03:51:05.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:51:05.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/ts-1/wal
03:51:05.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/ts-1/wal
03:51:05.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/ts-1/data/data
03:51:05.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:51:05.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:51:05.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:51:05.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:51:05.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:51:05.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:51:05.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:51:05.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:51:05.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:51:05.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:51:05.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:51:05.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:05.954676 4004 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:51:05.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:05.956512 4004 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:51:05.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:05.957121 4004 kserver.cc:163] Server-wide thread pool size limit: 3276
03:51:05.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:05.960004 4004 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
03:51:05.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:05.965193 4004 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:51:05.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:05.965411 4004 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
03:51:05.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:05.965622 4004 ts_tablet_manager.cc:610] Registered 0 tablets
03:51:05.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:05.965742 4004 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
03:51:06.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:06.166921 4004 rpc_server.cc:307] RPC server started. Bound to: 127.3.122.194:38019
03:51:06.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:06.167063 4137 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.3.122.194:38019 every 8 connection(s)
03:51:06.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:06.171087 4004 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/ts-1/data/info.pb
03:51:06.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:06.177325 3563 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskaXaVUt/build/tsan/bin/kudu as pid 4004
03:51:06.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:06.177911 3563 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/ts-1/wal/instance
03:51:06.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:06.238291 3563 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskaXaVUt/build/tsan/bin/kudu
03:51:06.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskaXaVUt/build/tsan/bin/kudu
03:51:06.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/ts-2/wal
03:51:06.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/ts-2/data
03:51:06.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:51:06.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:51:06.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:51:06.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:51:06.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:51:06.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:51:06.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/ts-2/logs
03:51:06.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/ts-2/data/info.pb
03:51:06.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:51:06.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:51:06.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:51:06.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:51:06.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:51:06.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:51:06.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:51:06.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:51:06.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:51:06.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:51:06.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.122.195:0
03:51:06.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.122.195
03:51:06.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.122.195
03:51:06.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:51:06.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.122.254:42965,127.3.122.253:35383,127.3.122.252:34035
03:51:06.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.122.212:36727
03:51:06.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:51:06.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:51:06.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
03:51:06.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:06.271836 4140 heartbeater.cc:347] Connected to a master server at 127.3.122.254:42965
03:51:06.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:06.272361 4140 heartbeater.cc:464] Registering TS with master...
03:51:06.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:06.273661 4140 heartbeater.cc:511] Master 127.3.122.254:42965 requested a full tablet report, sending...
03:51:06.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:06.274478 4139 heartbeater.cc:347] Connected to a master server at 127.3.122.253:35383
03:51:06.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:06.274820 4139 heartbeater.cc:464] Registering TS with master...
03:51:06.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:06.275777 4138 heartbeater.cc:347] Connected to a master server at 127.3.122.252:34035
03:51:06.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:06.275770 4139 heartbeater.cc:511] Master 127.3.122.253:35383 requested a full tablet report, sending...
03:51:06.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:06.276175 4138 heartbeater.cc:464] Registering TS with master...
03:51:06.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:06.276988 4138 heartbeater.cc:511] Master 127.3.122.252:34035 requested a full tablet report, sending...
03:51:06.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:06.278211 3607 ts_manager.cc:194] Registered new tserver with Master: ccfd9c961c50436dade821d75b3d9433 (127.3.122.194:38019)
03:51:06.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:06.281118 3764 ts_manager.cc:194] Registered new tserver with Master: ccfd9c961c50436dade821d75b3d9433 (127.3.122.194:38019)
03:51:06.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:06.283098 3683 ts_manager.cc:194] Registered new tserver with Master: ccfd9c961c50436dade821d75b3d9433 (127.3.122.194:38019)
03:51:06.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:06.283372 3764 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.3.122.194:59629
03:51:06.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:06.594559 4148 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:51:06.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:06.595106 4148 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:51:06.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:06.595626 4148 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:51:06.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:06.627235 4148 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:51:06.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:06.628160 4148 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.3.122.195
03:51:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:06.664285 4148 tablet_server_runner.cc:78] Tablet server non-default flags:
03:51:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:51:06.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.122.212:36727
03:51:06.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:51:06.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:51:06.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/ts-2/data
03:51:06.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/ts-2/wal
03:51:06.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:51:06.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:51:06.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.122.195:0
03:51:06.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:51:06.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:51:06.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:51:06.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/ts-2/data/info.pb
03:51:06.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.122.195
03:51:06.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:51:06.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.122.254:42965,127.3.122.253:35383,127.3.122.252:34035
03:51:06.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:51:06.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:51:06.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:51:06.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:51:06.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:51:06.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.122.195
03:51:06.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/ts-2/logs
03:51:06.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:51:06.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:51:06.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:51:06.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:51:06.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:51:06.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:51:06.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:51:06.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 02:49:50 UTC on ccycloud.quasar-twsplx.root.comops.site
03:51:06.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4675617
03:51:06.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:51:06.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:06.665920 4148 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:51:06.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:06.667902 4148 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:51:06.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:06.691758 4155 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:51:07.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:07.288059 4138 heartbeater.cc:503] Master 127.3.122.252:34035 was elected leader, sending a full tablet report...
03:51:07.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:06.692035 4157 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:51:07.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:06.696707 4148 server_base.cc:1034] running on GCE node
03:51:07.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:06.695400 4154 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:51:07.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:07.959285 4148 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:51:07.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:07.961781 4148 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:51:07.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:07.963549 4148 hybrid_clock.cc:648] HybridClock initialized: now 1790481067963509 us; error 49 us; skew 500 ppm
03:51:07.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:07.964356 4148 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:51:07.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:07.976595 4148 webserver.cc:460] Webserver started at http://127.3.122.195:33491/ using document root <none> and password file <none>
03:51:07.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:07.977562 4148 fs_manager.cc:359] Metadata directory not provided
03:51:07.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:07.977846 4148 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:51:07.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:07.978425 4148 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:51:07.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:07.984922 4148 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/ts-2/data/instance:
03:51:07.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "735576e14c2047d487af0d64d0cf15cb"
03:51:07.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 03:51:07 on dist-test-slave-c26w"
03:51:07.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:07.986407 4148 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/ts-2/wal/instance:
03:51:07.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "735576e14c2047d487af0d64d0cf15cb"
03:51:07.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 03:51:07 on dist-test-slave-c26w"
03:51:07.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:07.993589 4148 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.000s
03:51:07.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:07.999485 4165 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:51:08.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:08.000711 4148 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
03:51:08.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:08.001147 4148 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/ts-2/data,/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/ts-2/wal
03:51:08.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "735576e14c2047d487af0d64d0cf15cb"
03:51:08.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 03:51:07 on dist-test-slave-c26w"
03:51:08.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:08.001528 4148 fs_report.cc:383] FS layout report
03:51:08.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:51:08.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/ts-2/wal
03:51:08.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/ts-2/wal
03:51:08.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/ts-2/data/data
03:51:08.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:51:08.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:51:08.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:51:08.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:51:08.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:51:08.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:51:08.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:51:08.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:51:08.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:51:08.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:51:08.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:51:08.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:08.040966 4148 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:51:08.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:08.042819 4148 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:51:08.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:08.043462 4148 kserver.cc:163] Server-wide thread pool size limit: 3276
03:51:08.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:08.046350 4148 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
03:51:08.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:08.051244 4148 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:51:08.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:08.051546 4148 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
03:51:08.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:08.051879 4148 ts_tablet_manager.cc:610] Registered 0 tablets
03:51:08.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:08.052141 4148 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
03:51:08.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:08.223405 4148 rpc_server.cc:307] RPC server started. Bound to: 127.3.122.195:40873
03:51:08.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:08.223502 4277 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.3.122.195:40873 every 8 connection(s)
03:51:08.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:08.226233 4148 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/ts-2/data/info.pb
03:51:08.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:08.232172 3563 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskaXaVUt/build/tsan/bin/kudu as pid 4148
03:51:08.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:08.232661 3563 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster9053287806291578113/ts-2/wal/instance
03:51:08.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:08.292208 4280 heartbeater.cc:347] Connected to a master server at 127.3.122.254:42965
03:51:08.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:08.292747 4280 heartbeater.cc:464] Registering TS with master...
03:51:08.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:08.294477 4280 heartbeater.cc:511] Master 127.3.122.254:42965 requested a full tablet report, sending...
03:51:08.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:08.296417 4278 heartbeater.cc:347] Connected to a master server at 127.3.122.252:34035
03:51:08.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:08.296792 4278 heartbeater.cc:464] Registering TS with master...
03:51:08.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:08.297623 4278 heartbeater.cc:511] Master 127.3.122.252:34035 requested a full tablet report, sending...
03:51:08.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:08.299877 4279 heartbeater.cc:347] Connected to a master server at 127.3.122.253:35383
03:51:08.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:08.299930 3607 ts_manager.cc:194] Registered new tserver with Master: 735576e14c2047d487af0d64d0cf15cb (127.3.122.195:40873)
03:51:08.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:08.300173 4279 heartbeater.cc:464] Registering TS with master...
03:51:08.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:08.300650 3764 ts_manager.cc:194] Registered new tserver with Master: 735576e14c2047d487af0d64d0cf15cb (127.3.122.195:40873)
03:51:08.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:08.300976 4279 heartbeater.cc:511] Master 127.3.122.253:35383 requested a full tablet report, sending...
03:51:08.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:08.302153 3764 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.3.122.195:37725
03:51:08.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:08.304211 3683 ts_manager.cc:194] Registered new tserver with Master: 735576e14c2047d487af0d64d0cf15cb (127.3.122.195:40873)
03:51:08.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:08.312736 3563 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
03:51:08.324 [DEBUG - main] (MiniKuduCluster.java:195) Response:
03:51:08.326 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
03:51:08.339 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.3.122.254"
port: 42965
}
bound_http_address {
host: "127.3.122.254"
port: 41107
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.3.122.253"
port: 35383
}
bound_http_address {
host: "127.3.122.253"
port: 44633
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.3.122.252"
port: 34035
}
bound_http_address {
host: "127.3.122.252"
port: 33621
}
}
}
03:51:08.345 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
03:51:08.349 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.3.122.193"
port: 37293
}
bound_http_address {
host: "127.3.122.193"
port: 44319
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.3.122.194"
port: 38019
}
bound_http_address {
host: "127.3.122.194"
port: 38553
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.3.122.195"
port: 40873
}
bound_http_address {
host: "127.3.122.195"
port: 33491
}
}
}
03:51:08.349 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
03:51:08.546 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
03:51:08.670 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskaXaVUt/test-tmp
03:51:08.671 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
03:51:08.671 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskaXaVUt/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
03:51:08.674 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
03:51:09.050 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:51:09.050 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
03:51:09.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:09.053386 4289 mini_oidc.cc:152] Starting JWKS server
03:51:09.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:09.057018 4289 webserver.cc:460] Webserver started at http://127.0.0.1:36253/ using document root <none> and password file <none>
03:51:09.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:09.057467 4289 mini_oidc.cc:189] Starting OIDC Discovery server
03:51:09.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:09.059638 4289 webserver.cc:460] Webserver started at http://127.0.0.1:38527/ using document root <none> and password file <none>
03:51:09.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
03:51:09.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/krb5kdc/principal' for realm 'KRBTEST.COM',
03:51:09.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
03:51:09.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 03:51:09 dist-test-slave-c26w krb5kdc[4301](info): setting up network...
03:51:09.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
03:51:09.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 03:51:09 dist-test-slave-c26w krb5kdc[4301](info): set up 2 sockets
03:51:09.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 03:51:09 dist-test-slave-c26w krb5kdc[4301](info): commencing operation
03:51:09.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
03:51:09.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:09.311026 4278 heartbeater.cc:503] Master 127.3.122.252:34035 was elected leader, sending a full tablet report...
03:51:10.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:10.192543 4289 mini_kdc.cc:121] Time spent starting KDC: real 1.107s user 0.007s sys 0.015s
03:51:10.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
03:51:10.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
03:51:10.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
03:51:10.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
03:51:10.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
03:51:10.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
03:51:10.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
03:51:10.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
03:51:10.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
03:51:10.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
03:51:10.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/krb5kdc/test-user.keytab.
03:51:10.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/krb5kdc/test-user.keytab.
03:51:10.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 03:51:10 dist-test-slave-c26w krb5kdc[4301](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790481070, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:51:10.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
03:51:10.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T03:51:10Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
03:51:10.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T03:51:10Z Disabled control of system clock
03:51:10.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.4.48.126@KRBTEST.COM; defaulting to no policy
03:51:10.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:51:10.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.4.48.126@KRBTEST.COM" created.
03:51:10.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:51:10.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.4.48.126 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/krb5kdc/kudu_127.4.48.126.keytab.
03:51:10.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.4.48.126 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/krb5kdc/kudu_127.4.48.126.keytab.
03:51:10.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:10.410076 4289 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskaXaVUt/build/tsan/bin/kudu
03:51:10.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskaXaVUt/build/tsan/bin/kudu
03:51:10.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-0/wal
03:51:10.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-0/data
03:51:10.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:51:10.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:51:10.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:51:10.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:51:10.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:51:10.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:51:10.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-0/logs
03:51:10.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-0/data/info.pb
03:51:10.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:51:10.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:51:10.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:51:10.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:51:10.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:51:10.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:51:10.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:51:10.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:51:10.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:51:10.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:51:10.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:51:10.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:51:10.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.48.126:43853
03:51:10.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.48.126
03:51:10.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:51:10.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/krb5kdc/kudu_127.4.48.126.keytab
03:51:10.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.48.126
03:51:10.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:51:10.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:51:10.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:51:10.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.48.84:43463
03:51:10.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:51:10.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:51:10.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:51:10.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:51:10.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.48.126:43853,127.4.48.125:35747,127.4.48.124:33343
03:51:10.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:51:10.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36253/jwks.json
03:51:10.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:51:10.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:10.740724 4315 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:51:10.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:10.741350 4315 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:51:10.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:10.741914 4315 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:51:10.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:10.775200 4315 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:51:10.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:10.775442 4315 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:51:10.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:10.775702 4315 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:51:10.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:10.775897 4315 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:51:10.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:10.776118 4315 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:51:10.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:10.776304 4315 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:51:10.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:10.776439 4315 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36253/jwks.json
03:51:10.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:10.814688 4315 master_runner.cc:386] Master server non-default flags:
03:51:10.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:51:10.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.48.84:43463
03:51:10.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:51:10.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:51:10.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-0/data
03:51:10.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-0/wal
03:51:10.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:51:10.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:51:10.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.48.126:43853,127.4.48.125:35747,127.4.48.124:33343
03:51:10.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/krb5kdc/kudu_127.4.48.126.keytab
03:51:10.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.48.126
03:51:10.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:51:10.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:51:10.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:51:10.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.48.126:43853
03:51:10.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:51:10.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:51:10.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:51:10.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36253/jwks.json
03:51:10.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:51:10.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:51:10.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:51:10.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:51:10.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:51:10.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-0/data/info.pb
03:51:10.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.48.126
03:51:10.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:51:10.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:51:10.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:51:10.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:51:10.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:51:10.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:51:10.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-0/logs
03:51:10.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:51:10.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:51:10.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:51:10.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:51:10.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:51:10.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:51:10.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:51:10.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 02:49:50 UTC on ccycloud.quasar-twsplx.root.comops.site
03:51:10.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4675617
03:51:10.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:51:10.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:10.816144 4315 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:51:10.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:10.817488 4315 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:51:10.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:10.840056 4321 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:51:12.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:10.840134 4324 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:51:12.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:10.844132 4322 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:51:12.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:12.187920 4323 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1350 milliseconds
03:51:12.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:12.187999 4315 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:51:12.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:12.191623 4315 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:51:12.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:12.195168 4315 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:51:12.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:12.196652 4315 hybrid_clock.cc:648] HybridClock initialized: now 1790481072196616 us; error 55 us; skew 500 ppm
03:51:12.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:12.197515 4315 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:51:12.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 03:51:12 dist-test-slave-c26w krb5kdc[4301](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790481072, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.48.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:51:12.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:12.204298 4315 init.cc:377] Logged in from keytab as kudu/127.4.48.126@KRBTEST.COM (short username kudu)
03:51:12.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:12.210829 4315 webserver.cc:460] Webserver started at http://127.4.48.126:36597/ using document root <none> and password file <none>
03:51:12.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:12.211946 4315 fs_manager.cc:359] Metadata directory not provided
03:51:12.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:12.212209 4315 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:51:12.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:12.212733 4315 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:51:12.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:12.217680 4315 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-0/data/instance:
03:51:12.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3d31e45829e34488a283fca5aa119876"
03:51:12.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 03:51:12 on dist-test-slave-c26w"
03:51:12.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:12.219476 4315 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-0/wal/instance:
03:51:12.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3d31e45829e34488a283fca5aa119876"
03:51:12.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 03:51:12 on dist-test-slave-c26w"
03:51:12.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:12.228312 4315 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.007s sys 0.001s
03:51:12.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:12.236052 4332 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:51:12.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:12.237381 4315 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.002s sys 0.003s
03:51:12.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:12.237697 4315 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-0/data,/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-0/wal
03:51:12.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3d31e45829e34488a283fca5aa119876"
03:51:12.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 03:51:12 on dist-test-slave-c26w"
03:51:12.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:12.238030 4315 fs_report.cc:383] FS layout report
03:51:12.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:51:12.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-0/wal
03:51:12.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-0/wal
03:51:12.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-0/data/data
03:51:12.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:51:12.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:51:12.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:51:12.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:51:12.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:51:12.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:51:12.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:51:12.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:51:12.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:51:12.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:51:12.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:51:12.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:12.301631 4315 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:51:12.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:12.309463 4315 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:51:12.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:12.310556 4315 kserver.cc:163] Server-wide thread pool size limit: 3276
03:51:12.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:12.401343 4315 rpc_server.cc:307] RPC server started. Bound to: 127.4.48.126:43853
03:51:12.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:12.401433 4383 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.48.126:43853 every 8 connection(s)
03:51:12.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:12.404348 4315 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-0/data/info.pb
03:51:12.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:12.410382 4384 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:51:12.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:12.411761 4289 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskaXaVUt/build/tsan/bin/kudu as pid 4315
03:51:12.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:12.412334 4289 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-0/wal/instance
03:51:12.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:12.426592 4384 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.48.126" port: 43853 } has no permanent_uuid. Determining permanent_uuid...
03:51:12.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.4.48.125@KRBTEST.COM; defaulting to no policy
03:51:12.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:51:12.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.4.48.125@KRBTEST.COM" created.
03:51:12.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 03:51:12 dist-test-slave-c26w krb5kdc[4301](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790481072, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.48.126@KRBTEST.COM for kudu/127.4.48.126@KRBTEST.COM
03:51:12.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:51:12.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.4.48.125 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/krb5kdc/kudu_127.4.48.125.keytab.
03:51:12.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.4.48.125 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/krb5kdc/kudu_127.4.48.125.keytab.
03:51:12.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:12.463447 4289 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskaXaVUt/build/tsan/bin/kudu
03:51:12.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskaXaVUt/build/tsan/bin/kudu
03:51:12.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-1/wal
03:51:12.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-1/data
03:51:12.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:51:12.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:51:12.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:51:12.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:51:12.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:51:12.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:51:12.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-1/logs
03:51:12.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-1/data/info.pb
03:51:12.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:51:12.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:51:12.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:51:12.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:51:12.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:51:12.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:51:12.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:51:12.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:51:12.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:51:12.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:51:12.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:51:12.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:51:12.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.48.125:35747
03:51:12.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.48.125
03:51:12.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:51:12.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/krb5kdc/kudu_127.4.48.125.keytab
03:51:12.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.48.125
03:51:12.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:51:12.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:51:12.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:51:12.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.48.84:43463
03:51:12.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:51:12.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:51:12.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:51:12.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:51:12.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.48.126:43853,127.4.48.125:35747,127.4.48.124:33343
03:51:12.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:51:12.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36253/jwks.json
03:51:12.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:51:12.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:12.471498 4384 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.48.125" port: 35747 } has no permanent_uuid. Determining permanent_uuid...
03:51:12.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:12.474325 4336 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.48.125:35747: connect: Connection refused (error 111)
03:51:12.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:12.478466 4384 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.48.125:35747: Network error: Client connection negotiation failed: client connection to 127.4.48.125:35747: connect: Connection refused (error 111)
03:51:12.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:12.527920 4384 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.48.125" port: 35747 } attempt: 1
03:51:12.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:12.532215 4384 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.48.125:35747: Network error: Client connection negotiation failed: client connection to 127.4.48.125:35747: connect: Connection refused (error 111)
03:51:12.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:12.600662 4384 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.48.125" port: 35747 } attempt: 2
03:51:12.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:12.604933 4384 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.48.125:35747: Network error: Client connection negotiation failed: client connection to 127.4.48.125:35747: connect: Connection refused (error 111)
03:51:12.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:12.696409 4384 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.48.125" port: 35747 } attempt: 3
03:51:12.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:12.700987 4384 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.48.125:35747: Network error: Client connection negotiation failed: client connection to 127.4.48.125:35747: connect: Connection refused (error 111)
03:51:12.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:12.801316 4390 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:51:12.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:12.801893 4390 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:51:12.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:12.802452 4390 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:51:12.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:12.835846 4390 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:51:12.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:12.836145 4390 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:51:12.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:12.836573 4390 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:51:12.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:12.836895 4390 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:51:12.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:12.837185 4390 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:51:12.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:12.837473 4390 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:51:12.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:12.837693 4390 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36253/jwks.json
03:51:12.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:12.844466 4384 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.48.125" port: 35747 } attempt: 4
03:51:12.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:12.849416 4384 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.48.125:35747: Network error: Client connection negotiation failed: client connection to 127.4.48.125:35747: connect: Connection refused (error 111)
03:51:12.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:12.891393 4390 master_runner.cc:386] Master server non-default flags:
03:51:12.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:51:12.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.48.84:43463
03:51:12.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:51:12.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:51:12.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-1/data
03:51:12.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-1/wal
03:51:12.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:51:12.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:51:12.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.48.126:43853,127.4.48.125:35747,127.4.48.124:33343
03:51:12.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/krb5kdc/kudu_127.4.48.125.keytab
03:51:12.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.48.125
03:51:12.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:51:12.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:51:12.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:51:12.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.48.125:35747
03:51:12.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:51:12.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:51:12.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:51:12.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36253/jwks.json
03:51:12.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:51:12.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:51:12.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:51:12.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:51:12.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:51:12.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-1/data/info.pb
03:51:12.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.48.125
03:51:12.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:51:12.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:51:12.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:51:12.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:51:12.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:51:12.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:51:12.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-1/logs
03:51:12.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:51:12.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:51:12.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:51:12.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:51:12.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:51:12.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:51:12.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:51:12.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 02:49:50 UTC on ccycloud.quasar-twsplx.root.comops.site
03:51:12.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4675617
03:51:12.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:51:12.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:12.892860 4390 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:51:12.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:12.894110 4390 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:51:12.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:12.922274 4398 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:51:13.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:13.148838 4384 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.48.125" port: 35747 } attempt: 5
03:51:13.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:13.153950 4384 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.48.125:35747: Network error: Client connection negotiation failed: client connection to 127.4.48.125:35747: connect: Connection refused (error 111)
03:51:13.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:13.701542 4384 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.48.125" port: 35747 } attempt: 6
03:51:13.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:13.708962 4384 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.48.125:35747: Network error: Client connection negotiation failed: client connection to 127.4.48.125:35747: connect: Connection refused (error 111)
03:51:14.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:12.926178 4401 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:51:14.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:12.925969 4397 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:51:14.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:12.926362 4390 server_base.cc:1034] running on GCE node
03:51:14.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:14.185707 4390 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:51:14.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:14.188287 4390 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:51:14.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:14.189679 4390 hybrid_clock.cc:648] HybridClock initialized: now 1790481074189645 us; error 43 us; skew 500 ppm
03:51:14.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:14.190551 4390 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:51:14.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 03:51:14 dist-test-slave-c26w krb5kdc[4301](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790481074, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.48.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:51:14.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:14.200132 4390 init.cc:377] Logged in from keytab as kudu/127.4.48.125@KRBTEST.COM (short username kudu)
03:51:14.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:14.205271 4390 webserver.cc:460] Webserver started at http://127.4.48.125:44665/ using document root <none> and password file <none>
03:51:14.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:14.206328 4390 fs_manager.cc:359] Metadata directory not provided
03:51:14.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:14.206576 4390 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:51:14.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:14.207131 4390 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:51:14.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:14.212404 4390 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-1/data/instance:
03:51:14.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ead70800fa0e488da8e775fa5212d242"
03:51:14.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 03:51:14 on dist-test-slave-c26w"
03:51:14.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:14.213759 4390 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-1/wal/instance:
03:51:14.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ead70800fa0e488da8e775fa5212d242"
03:51:14.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 03:51:14 on dist-test-slave-c26w"
03:51:14.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:14.222213 4390 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.005s sys 0.004s
03:51:14.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:14.230571 4413 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:51:14.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:14.232054 4390 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.002s sys 0.001s
03:51:14.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:14.232467 4390 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-1/data,/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-1/wal
03:51:14.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ead70800fa0e488da8e775fa5212d242"
03:51:14.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 03:51:14 on dist-test-slave-c26w"
03:51:14.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:14.232940 4390 fs_report.cc:383] FS layout report
03:51:14.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:51:14.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-1/wal
03:51:14.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-1/wal
03:51:14.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-1/data/data
03:51:14.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:51:14.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:51:14.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:51:14.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:51:14.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:51:14.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:51:14.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:51:14.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:51:14.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:51:14.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:51:14.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:51:14.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:14.275174 4390 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:51:14.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:14.281492 4390 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:51:14.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:14.281908 4390 kserver.cc:163] Server-wide thread pool size limit: 3276
03:51:14.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:14.382593 4390 rpc_server.cc:307] RPC server started. Bound to: 127.4.48.125:35747
03:51:14.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:14.382678 4464 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.48.125:35747 every 8 connection(s)
03:51:14.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:14.385190 4390 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-1/data/info.pb
03:51:14.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:14.387771 4289 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskaXaVUt/build/tsan/bin/kudu as pid 4390
03:51:14.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:14.388238 4289 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-1/wal/instance
03:51:14.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:14.392478 4465 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:51:14.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.4.48.124@KRBTEST.COM; defaulting to no policy
03:51:14.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:14.406912 4465 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.48.126" port: 43853 } has no permanent_uuid. Determining permanent_uuid...
03:51:14.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:51:14.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.4.48.124@KRBTEST.COM" created.
03:51:14.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 03:51:14 dist-test-slave-c26w krb5kdc[4301](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790481074, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.48.125@KRBTEST.COM for kudu/127.4.48.126@KRBTEST.COM
03:51:14.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:51:14.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.4.48.124 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/krb5kdc/kudu_127.4.48.124.keytab.
03:51:14.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.4.48.124 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/krb5kdc/kudu_127.4.48.124.keytab.
03:51:14.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:14.445212 4289 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskaXaVUt/build/tsan/bin/kudu
03:51:14.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskaXaVUt/build/tsan/bin/kudu
03:51:14.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-2/wal
03:51:14.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-2/data
03:51:14.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:51:14.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:51:14.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:51:14.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:51:14.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:51:14.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:51:14.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-2/logs
03:51:14.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-2/data/info.pb
03:51:14.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:51:14.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:51:14.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:51:14.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:51:14.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:51:14.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:51:14.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:51:14.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:51:14.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:51:14.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:51:14.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:51:14.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:51:14.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.48.124:33343
03:51:14.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.48.124
03:51:14.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:51:14.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/krb5kdc/kudu_127.4.48.124.keytab
03:51:14.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.48.124
03:51:14.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:51:14.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:51:14.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:51:14.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.48.84:43463
03:51:14.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:51:14.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:51:14.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:51:14.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:51:14.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.48.126:43853,127.4.48.125:35747,127.4.48.124:33343
03:51:14.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:51:14.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36253/jwks.json
03:51:14.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:51:14.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:14.450578 4465 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.48.125" port: 35747 } has no permanent_uuid. Determining permanent_uuid...
03:51:14.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 03:51:14 dist-test-slave-c26w krb5kdc[4301](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790481074, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.48.125@KRBTEST.COM for kudu/127.4.48.125@KRBTEST.COM
03:51:14.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:14.482599 4465 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.48.124" port: 33343 } has no permanent_uuid. Determining permanent_uuid...
03:51:14.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:14.484880 4414 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.48.124:33343: connect: Connection refused (error 111)
03:51:14.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:14.489302 4465 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.48.124:33343: Network error: Client connection negotiation failed: client connection to 127.4.48.124:33343: connect: Connection refused (error 111)
03:51:14.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:14.538803 4465 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.48.124" port: 33343 } attempt: 1
03:51:14.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:14.543022 4465 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.48.124:33343: Network error: Client connection negotiation failed: client connection to 127.4.48.124:33343: connect: Connection refused (error 111)
03:51:14.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:14.611485 4465 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.48.124" port: 33343 } attempt: 2
03:51:14.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:14.615473 4465 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.48.124:33343: Network error: Client connection negotiation failed: client connection to 127.4.48.124:33343: connect: Connection refused (error 111)
03:51:14.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:14.707096 4465 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.48.124" port: 33343 } attempt: 3
03:51:14.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:14.710937 4465 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.48.124:33343: Network error: Client connection negotiation failed: client connection to 127.4.48.124:33343: connect: Connection refused (error 111)
03:51:14.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:14.769632 4384 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.48.125" port: 35747 } attempt: 7
03:51:14.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:14.776626 4471 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:51:14.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:14.777420 4471 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:51:14.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:14.778441 4471 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:51:14.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 03:51:14 dist-test-slave-c26w krb5kdc[4301](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790481072, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.48.126@KRBTEST.COM for kudu/127.4.48.125@KRBTEST.COM
03:51:14.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:14.795498 4384 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.48.124" port: 33343 } has no permanent_uuid. Determining permanent_uuid...
03:51:14.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:14.801260 4384 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.48.124:33343: Network error: Client connection negotiation failed: client connection to 127.4.48.124:33343: connect: Connection refused (error 111)
03:51:14.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:14.816511 4471 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:51:14.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:14.816730 4471 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:51:14.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:14.816984 4471 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:51:14.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:14.817184 4471 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:51:14.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:14.817368 4471 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:51:14.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:14.817541 4471 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:51:14.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:14.817672 4471 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36253/jwks.json
03:51:14.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:14.854319 4465 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.48.124" port: 33343 } attempt: 4
03:51:14.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:14.858950 4465 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.48.124:33343: Network error: Client connection negotiation failed: client connection to 127.4.48.124:33343: connect: Connection refused (error 111)
03:51:14.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:14.859671 4384 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.48.124" port: 33343 } attempt: 1
03:51:14.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:14.863648 4384 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.48.124:33343: Network error: Client connection negotiation failed: client connection to 127.4.48.124:33343: connect: Connection refused (error 111)
03:51:14.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:14.865410 4471 master_runner.cc:386] Master server non-default flags:
03:51:14.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:51:14.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.48.84:43463
03:51:14.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:51:14.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:51:14.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-2/data
03:51:14.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-2/wal
03:51:14.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:51:14.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:51:14.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.48.126:43853,127.4.48.125:35747,127.4.48.124:33343
03:51:14.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/krb5kdc/kudu_127.4.48.124.keytab
03:51:14.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.48.124
03:51:14.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:51:14.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:51:14.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:51:14.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.48.124:33343
03:51:14.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:51:14.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:51:14.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:51:14.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36253/jwks.json
03:51:14.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:51:14.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:51:14.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:51:14.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:51:14.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:51:14.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-2/data/info.pb
03:51:14.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.48.124
03:51:14.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:51:14.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:51:14.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:51:14.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:51:14.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:51:14.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:51:14.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-2/logs
03:51:14.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:51:14.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:51:14.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:51:14.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:51:14.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:51:14.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:51:14.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:51:14.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 02:49:50 UTC on ccycloud.quasar-twsplx.root.comops.site
03:51:14.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4675617
03:51:14.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:51:14.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:14.866542 4471 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:51:14.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:14.867615 4471 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:51:14.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:14.892174 4484 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:51:14.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:14.945147 4384 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.48.124" port: 33343 } attempt: 2
03:51:14.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:14.950912 4384 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.48.124:33343: Network error: Client connection negotiation failed: client connection to 127.4.48.124:33343: connect: Connection refused (error 111)
03:51:15.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:15.036433 4384 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.48.124" port: 33343 } attempt: 3
03:51:15.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:15.042011 4384 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.48.124:33343: Network error: Client connection negotiation failed: client connection to 127.4.48.124:33343: connect: Connection refused (error 111)
03:51:15.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:15.158429 4465 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.48.124" port: 33343 } attempt: 5
03:51:15.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:15.167266 4465 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.48.124:33343: Network error: Client connection negotiation failed: client connection to 127.4.48.124:33343: connect: Connection refused (error 111)
03:51:15.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:15.182447 4384 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.48.124" port: 33343 } attempt: 4
03:51:15.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:15.185973 4384 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.48.124:33343: Network error: Client connection negotiation failed: client connection to 127.4.48.124:33343: connect: Connection refused (error 111)
03:51:15.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:15.469762 4384 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.48.124" port: 33343 } attempt: 5
03:51:15.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:15.474592 4384 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.48.124:33343: Network error: Client connection negotiation failed: client connection to 127.4.48.124:33343: connect: Connection refused (error 111)
03:51:15.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:15.714895 4465 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.48.124" port: 33343 } attempt: 6
03:51:15.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:15.723568 4465 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.48.124:33343: Network error: Client connection negotiation failed: client connection to 127.4.48.124:33343: connect: Connection refused (error 111)
03:51:16.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:16.027063 4384 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.48.124" port: 33343 } attempt: 6
03:51:16.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:16.034683 4384 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.48.124:33343: Network error: Client connection negotiation failed: client connection to 127.4.48.124:33343: connect: Connection refused (error 111)
03:51:16.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:14.892197 4482 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:51:16.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:14.894304 4471 server_base.cc:1034] running on GCE node
03:51:16.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:14.895082 4481 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:51:16.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:16.154014 4471 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:51:16.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:16.157567 4471 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:51:16.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:16.159027 4471 hybrid_clock.cc:648] HybridClock initialized: now 1790481076158946 us; error 95 us; skew 500 ppm
03:51:16.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:16.159916 4471 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:51:16.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 03:51:16 dist-test-slave-c26w krb5kdc[4301](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790481076, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.48.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:51:16.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:16.165977 4471 init.cc:377] Logged in from keytab as kudu/127.4.48.124@KRBTEST.COM (short username kudu)
03:51:16.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:16.171861 4471 webserver.cc:460] Webserver started at http://127.4.48.124:43725/ using document root <none> and password file <none>
03:51:16.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:16.172995 4471 fs_manager.cc:359] Metadata directory not provided
03:51:16.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:16.173254 4471 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:51:16.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:16.173717 4471 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:51:16.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:16.178861 4471 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-2/data/instance:
03:51:16.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b08ed0a784ea44f7b4941b6eadaf4e13"
03:51:16.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 03:51:16 on dist-test-slave-c26w"
03:51:16.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:16.180274 4471 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-2/wal/instance:
03:51:16.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b08ed0a784ea44f7b4941b6eadaf4e13"
03:51:16.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 03:51:16 on dist-test-slave-c26w"
03:51:16.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:16.188489 4471 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.003s
03:51:16.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:16.194959 4497 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:51:16.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:16.196075 4471 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
03:51:16.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:16.196493 4471 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-2/data,/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-2/wal
03:51:16.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b08ed0a784ea44f7b4941b6eadaf4e13"
03:51:16.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 03:51:16 on dist-test-slave-c26w"
03:51:16.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:16.196880 4471 fs_report.cc:383] FS layout report
03:51:16.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:51:16.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-2/wal
03:51:16.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-2/wal
03:51:16.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-2/data/data
03:51:16.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:51:16.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:51:16.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:51:16.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:51:16.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:51:16.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:51:16.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:51:16.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:51:16.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:51:16.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:51:16.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:51:16.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:16.239976 4471 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:51:16.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:16.246462 4471 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:51:16.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:16.247000 4471 kserver.cc:163] Server-wide thread pool size limit: 3276
03:51:16.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:16.343292 4471 rpc_server.cc:307] RPC server started. Bound to: 127.4.48.124:33343
03:51:16.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:16.343356 4548 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.48.124:33343 every 8 connection(s)
03:51:16.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:16.345978 4471 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-2/data/info.pb
03:51:16.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:16.352183 4549 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:51:16.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:16.353391 4289 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskaXaVUt/build/tsan/bin/kudu as pid 4471
03:51:16.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:16.353825 4289 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-2/wal/instance
03:51:16.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:16.367254 4549 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.48.126" port: 43853 } has no permanent_uuid. Determining permanent_uuid...
03:51:16.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 03:51:16 dist-test-slave-c26w krb5kdc[4301](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790481076, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.48.124@KRBTEST.COM for kudu/127.4.48.126@KRBTEST.COM
03:51:16.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 03:51:16 dist-test-slave-c26w krb5kdc[4301](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790481070, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.4.48.124@KRBTEST.COM
03:51:16.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:16.407764 4549 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.48.125" port: 35747 } has no permanent_uuid. Determining permanent_uuid...
03:51:16.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 03:51:16 dist-test-slave-c26w krb5kdc[4301](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790481076, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.48.124@KRBTEST.COM for kudu/127.4.48.125@KRBTEST.COM
03:51:16.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 03:51:16 dist-test-slave-c26w krb5kdc[4301](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790481070, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.4.48.125@KRBTEST.COM
03:51:16.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:16.438862 4549 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.48.124" port: 33343 } has no permanent_uuid. Determining permanent_uuid...
03:51:16.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 03:51:16 dist-test-slave-c26w krb5kdc[4301](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790481076, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.48.124@KRBTEST.COM for kudu/127.4.48.124@KRBTEST.COM
03:51:16.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 03:51:16 dist-test-slave-c26w krb5kdc[4301](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790481070, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.4.48.126@KRBTEST.COM
03:51:16.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:16.473834 4289 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
03:51:16.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:16.474131 4549 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13: Bootstrap starting.
03:51:16.484 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:51:16.485 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
03:51:16.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:16.481832 4549 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13: Neither blocks nor log segments found. Creating new log.
03:51:16.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:16.484386 4549 log.cc:824] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13: Log is configured to *not* fsync() on all Append() calls
03:51:16.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:16.489689 4549 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13: No bootstrap required, opened a new log
03:51:16.505 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
03:51:16.507 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
03:51:16.511 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.4.48.126"
port: 43853
}
bound_http_address {
host: "127.4.48.126"
port: 36597
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.4.48.125"
port: 35747
}
bound_http_address {
host: "127.4.48.125"
port: 44665
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.4.48.124"
port: 33343
}
bound_http_address {
host: "127.4.48.124"
port: 43725
}
}
}
03:51:16.511 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
03:51:16.512 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:51:16.512 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
03:51:16.514 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
03:51:16.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:16.515486 4549 raft_consensus.cc:348] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d31e45829e34488a283fca5aa119876" member_type: VOTER last_known_addr { host: "127.4.48.126" port: 43853 } } peers { permanent_uuid: "ead70800fa0e488da8e775fa5212d242" member_type: VOTER last_known_addr { host: "127.4.48.125" port: 35747 } } peers { permanent_uuid: "b08ed0a784ea44f7b4941b6eadaf4e13" member_type: VOTER last_known_addr { host: "127.4.48.124" port: 33343 } }
03:51:16.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:16.516260 4549 raft_consensus.cc:374] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:51:16.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:16.516563 4549 raft_consensus.cc:729] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b08ed0a784ea44f7b4941b6eadaf4e13, State: Initialized, Role: FOLLOWER
03:51:16.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:16.517427 4549 consensus_queue.cc:260] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d31e45829e34488a283fca5aa119876" member_type: VOTER last_known_addr { host: "127.4.48.126" port: 43853 } } peers { permanent_uuid: "ead70800fa0e488da8e775fa5212d242" member_type: VOTER last_known_addr { host: "127.4.48.125" port: 35747 } } peers { permanent_uuid: "b08ed0a784ea44f7b4941b6eadaf4e13" member_type: VOTER last_known_addr { host: "127.4.48.124" port: 33343 } }
03:51:16.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:16.520998 4558 sys_catalog.cc:455] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d31e45829e34488a283fca5aa119876" member_type: VOTER last_known_addr { host: "127.4.48.126" port: 43853 } } peers { permanent_uuid: "ead70800fa0e488da8e775fa5212d242" member_type: VOTER last_known_addr { host: "127.4.48.125" port: 35747 } } peers { permanent_uuid: "b08ed0a784ea44f7b4941b6eadaf4e13" member_type: VOTER last_known_addr { host: "127.4.48.124" port: 33343 } } }
03:51:16.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:16.522101 4558 sys_catalog.cc:458] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13 [sys.catalog]: This master's current role is: FOLLOWER
03:51:16.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:16.522837 4549 sys_catalog.cc:564] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13 [sys.catalog]: configured and running, proceeding with master startup.
03:51:16.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 03:51:16 dist-test-slave-c26w krb5kdc[4301](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790481076, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:51:16.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
03:51:16.554 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:51:16.556 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/krb5kdc/krb5cc
03:51:16.571 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
03:51:16.572 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/krb5kdc/krb5cc
03:51:16.575 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
03:51:16.580 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
03:51:16.585 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
03:51:16.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:16.585186 4571 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:51:16.585 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
03:51:16.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:16.585522 4571 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:51:16.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:16.558797 4471 master_runner.cc:186] Error getting master registration for 127.4.48.126:43853: OK, instance_id { permanent_uuid: "3d31e45829e34488a283fca5aa119876" instance_seqno: 1790481072377895 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
03:51:16.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:16.606453 4471 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:51:16.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 03:51:16 dist-test-slave-c26w krb5kdc[4301](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790481076, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:51:16.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
03:51:16.625 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:51:16.625 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/krb5kdc/krb5cc
03:51:16.626 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
03:51:16.628 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.4.48.124:33343
03:51:16.629 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
03:51:16.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:16.630200 4289 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskaXaVUt/build/tsan/bin/kudu with pid 4471
03:51:16.651 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:51:16.651 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.4.48.126:43853
03:51:16.652 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
03:51:16.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:16.652439 4289 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskaXaVUt/build/tsan/bin/kudu with pid 4315
03:51:16.673 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:51:16.673 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.4.48.125:35747
03:51:16.673 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
03:51:16.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:16.673930 4289 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskaXaVUt/build/tsan/bin/kudu with pid 4390
03:51:16.694 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:51:16.695 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.4.48.124:33343
03:51:16.696 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
03:51:16.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:16.698235 4289 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskaXaVUt/build/tsan/bin/kudu
03:51:16.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskaXaVUt/build/tsan/bin/kudu
03:51:16.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-2/wal
03:51:16.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-2/data
03:51:16.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:51:16.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:51:16.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:51:16.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:51:16.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:51:16.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:51:16.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-2/logs
03:51:16.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-2/data/info.pb
03:51:16.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:51:16.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:51:16.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:51:16.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:51:16.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:51:16.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:51:16.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:51:16.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:51:16.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:51:16.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:51:16.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:51:16.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:51:16.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.48.124:33343
03:51:16.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.48.124
03:51:16.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43725
03:51:16.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/krb5kdc/kudu_127.4.48.124.keytab
03:51:16.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.48.124
03:51:16.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:51:16.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:51:16.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:51:16.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.48.84:43463
03:51:16.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:51:16.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:51:16.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:51:16.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:51:16.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.48.126:43853,127.4.48.125:35747,127.4.48.124:33343
03:51:16.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:51:16.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36253/jwks.json
03:51:16.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:51:17.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:17.010350 4576 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:51:17.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:17.010919 4576 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:51:17.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:17.011533 4576 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:51:17.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:17.045660 4576 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:51:17.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:17.045887 4576 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:51:17.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:17.046152 4576 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:51:17.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:17.046342 4576 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:51:17.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:17.046525 4576 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:51:17.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:17.046713 4576 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:51:17.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:17.046842 4576 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36253/jwks.json
03:51:17.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:17.083148 4576 master_runner.cc:386] Master server non-default flags:
03:51:17.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:51:17.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.48.84:43463
03:51:17.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:51:17.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:51:17.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-2/data
03:51:17.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-2/wal
03:51:17.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:51:17.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:51:17.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.48.126:43853,127.4.48.125:35747,127.4.48.124:33343
03:51:17.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/krb5kdc/kudu_127.4.48.124.keytab
03:51:17.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.48.124
03:51:17.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:51:17.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:51:17.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:51:17.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.48.124:33343
03:51:17.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:51:17.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:51:17.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:51:17.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36253/jwks.json
03:51:17.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:51:17.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:51:17.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:51:17.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:51:17.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:51:17.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-2/data/info.pb
03:51:17.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.48.124
03:51:17.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43725
03:51:17.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:51:17.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:51:17.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:51:17.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:51:17.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:51:17.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-2/logs
03:51:17.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:51:17.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:51:17.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:51:17.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:51:17.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:51:17.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:51:17.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:51:17.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 02:49:50 UTC on ccycloud.quasar-twsplx.root.comops.site
03:51:17.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4675617
03:51:17.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:51:17.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:17.084511 4576 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:51:17.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:17.085508 4576 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:51:17.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:17.102756 4583 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:51:18.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:17.102763 4585 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:51:18.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:17.104189 4576 server_base.cc:1034] running on GCE node
03:51:18.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:17.103789 4582 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:51:18.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:18.356678 4576 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:51:18.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:18.359889 4576 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:51:18.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:18.361373 4576 hybrid_clock.cc:648] HybridClock initialized: now 1790481078361318 us; error 59 us; skew 500 ppm
03:51:18.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:18.362286 4576 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:51:18.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 03:51:18 dist-test-slave-c26w krb5kdc[4301](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790481078, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.48.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:51:18.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:18.368517 4576 init.cc:377] Logged in from keytab as kudu/127.4.48.124@KRBTEST.COM (short username kudu)
03:51:18.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:18.373764 4576 webserver.cc:460] Webserver started at http://127.4.48.124:43725/ using document root <none> and password file <none>
03:51:18.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:18.374681 4576 fs_manager.cc:359] Metadata directory not provided
03:51:18.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:18.374966 4576 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:51:18.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:18.382802 4576 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.007s sys 0.000s
03:51:18.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:18.387355 4594 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:51:18.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:18.388477 4576 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
03:51:18.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:18.388846 4576 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-2/data,/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-2/wal
03:51:18.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b08ed0a784ea44f7b4941b6eadaf4e13"
03:51:18.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 03:51:16 on dist-test-slave-c26w"
03:51:18.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:18.391290 4576 fs_report.cc:383] FS layout report
03:51:18.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:51:18.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-2/wal
03:51:18.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-2/wal
03:51:18.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-2/data/data
03:51:18.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:51:18.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:51:18.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:51:18.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:51:18.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:51:18.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:51:18.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:51:18.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:51:18.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:51:18.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:51:18.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:51:18.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:18.440196 4576 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:51:18.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:18.446592 4576 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:51:18.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:18.447170 4576 kserver.cc:163] Server-wide thread pool size limit: 3276
03:51:18.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:18.520987 4576 rpc_server.cc:307] RPC server started. Bound to: 127.4.48.124:33343
03:51:18.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:18.521100 4645 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.48.124:33343 every 8 connection(s)
03:51:18.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:18.523893 4576 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-2/data/info.pb
03:51:18.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:18.531490 4289 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskaXaVUt/build/tsan/bin/kudu as pid 4576
03:51:18.532 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:51:18.532 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.4.48.126:43853
03:51:18.532 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
03:51:18.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:18.534163 4289 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskaXaVUt/build/tsan/bin/kudu
03:51:18.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskaXaVUt/build/tsan/bin/kudu
03:51:18.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-0/wal
03:51:18.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-0/data
03:51:18.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:51:18.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:51:18.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:51:18.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:51:18.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:51:18.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:51:18.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-0/logs
03:51:18.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-0/data/info.pb
03:51:18.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:51:18.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:51:18.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:51:18.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:51:18.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:51:18.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:51:18.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:51:18.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:51:18.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:51:18.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:51:18.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:51:18.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:51:18.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.48.126:43853
03:51:18.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.48.126
03:51:18.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36597
03:51:18.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/krb5kdc/kudu_127.4.48.126.keytab
03:51:18.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.48.126
03:51:18.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:51:18.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:51:18.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:51:18.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.48.84:43463
03:51:18.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:51:18.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:51:18.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:51:18.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:51:18.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.48.126:43853,127.4.48.125:35747,127.4.48.124:33343
03:51:18.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:51:18.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36253/jwks.json
03:51:18.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:51:18.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:18.535763 4646 sys_catalog.cc:263] Verifying existing consensus state
03:51:18.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:18.544399 4646 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13: Bootstrap starting.
03:51:18.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:18.576617 4646 log.cc:824] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13: Log is configured to *not* fsync() on all Append() calls
03:51:18.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:18.581120 4646 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
03:51:18.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:18.581790 4646 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13: Bootstrap complete.
03:51:18.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:18.606005 4646 raft_consensus.cc:348] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d31e45829e34488a283fca5aa119876" member_type: VOTER last_known_addr { host: "127.4.48.126" port: 43853 } } peers { permanent_uuid: "ead70800fa0e488da8e775fa5212d242" member_type: VOTER last_known_addr { host: "127.4.48.125" port: 35747 } } peers { permanent_uuid: "b08ed0a784ea44f7b4941b6eadaf4e13" member_type: VOTER last_known_addr { host: "127.4.48.124" port: 33343 } }
03:51:18.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:18.608630 4646 raft_consensus.cc:374] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:51:18.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:18.608841 4646 raft_consensus.cc:729] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b08ed0a784ea44f7b4941b6eadaf4e13, State: Initialized, Role: FOLLOWER
03:51:18.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:18.609870 4646 consensus_queue.cc:260] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d31e45829e34488a283fca5aa119876" member_type: VOTER last_known_addr { host: "127.4.48.126" port: 43853 } } peers { permanent_uuid: "ead70800fa0e488da8e775fa5212d242" member_type: VOTER last_known_addr { host: "127.4.48.125" port: 35747 } } peers { permanent_uuid: "b08ed0a784ea44f7b4941b6eadaf4e13" member_type: VOTER last_known_addr { host: "127.4.48.124" port: 33343 } }
03:51:18.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:18.612914 4650 sys_catalog.cc:455] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d31e45829e34488a283fca5aa119876" member_type: VOTER last_known_addr { host: "127.4.48.126" port: 43853 } } peers { permanent_uuid: "ead70800fa0e488da8e775fa5212d242" member_type: VOTER last_known_addr { host: "127.4.48.125" port: 35747 } } peers { permanent_uuid: "b08ed0a784ea44f7b4941b6eadaf4e13" member_type: VOTER last_known_addr { host: "127.4.48.124" port: 33343 } } }
03:51:18.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:18.614027 4650 sys_catalog.cc:458] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13 [sys.catalog]: This master's current role is: FOLLOWER
03:51:18.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:18.614638 4646 sys_catalog.cc:564] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13 [sys.catalog]: configured and running, proceeding with master startup.
03:51:18.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:18.647615 4661 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:51:18.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:18.648458 4661 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:51:18.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:18.642817 4576 master_runner.cc:186] Error getting master registration for 127.4.48.126:43853: Network error: Client connection negotiation failed: client connection to 127.4.48.126:43853: connect: Connection refused (error 111),
03:51:18.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:18.673418 4576 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:51:18.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:18.702576 4600 tablet.cc:2369] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:51:18.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:18.826143 4650 raft_consensus.cc:482] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:51:18.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:18.826957 4650 raft_consensus.cc:504] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d31e45829e34488a283fca5aa119876" member_type: VOTER last_known_addr { host: "127.4.48.126" port: 43853 } } peers { permanent_uuid: "ead70800fa0e488da8e775fa5212d242" member_type: VOTER last_known_addr { host: "127.4.48.125" port: 35747 } } peers { permanent_uuid: "b08ed0a784ea44f7b4941b6eadaf4e13" member_type: VOTER last_known_addr { host: "127.4.48.124" port: 33343 } }
03:51:18.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:18.831871 4595 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.48.126:43853: connect: Connection refused (error 111)
03:51:18.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:18.832626 4650 leader_election.cc:288] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3d31e45829e34488a283fca5aa119876 (127.4.48.126:43853), ead70800fa0e488da8e775fa5212d242 (127.4.48.125:35747)
03:51:18.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:18.846737 4595 leader_election.cc:334] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 3d31e45829e34488a283fca5aa119876 (127.4.48.126:43853): Network error: Client connection negotiation failed: client connection to 127.4.48.126:43853: connect: Connection refused (error 111)
03:51:18.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:18.851823 4598 leader_election.cc:334] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ead70800fa0e488da8e775fa5212d242 (127.4.48.125:35747): Network error: Client connection negotiation failed: client connection to 127.4.48.125:35747: connect: Connection refused (error 111)
03:51:18.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:18.852252 4598 leader_election.cc:302] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: b08ed0a784ea44f7b4941b6eadaf4e13; no voters: 3d31e45829e34488a283fca5aa119876, ead70800fa0e488da8e775fa5212d242
03:51:18.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:18.853085 4650 raft_consensus.cc:2726] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
03:51:18.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:18.895364 4648 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:51:18.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:18.896149 4648 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:51:18.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:18.897177 4648 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:51:18.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:18.953413 4648 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:51:18.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:18.954293 4648 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:51:18.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:18.954787 4648 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:51:18.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:18.955160 4648 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:51:18.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:18.955478 4648 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:51:18.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:18.955796 4648 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:51:18.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:18.956040 4648 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36253/jwks.json
03:51:19.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:19.008591 4648 master_runner.cc:386] Master server non-default flags:
03:51:19.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:51:19.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.48.84:43463
03:51:19.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:51:19.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:51:19.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-0/data
03:51:19.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-0/wal
03:51:19.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:51:19.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:51:19.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.48.126:43853,127.4.48.125:35747,127.4.48.124:33343
03:51:19.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/krb5kdc/kudu_127.4.48.126.keytab
03:51:19.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.48.126
03:51:19.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:51:19.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:51:19.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:51:19.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.48.126:43853
03:51:19.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:51:19.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:51:19.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:51:19.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36253/jwks.json
03:51:19.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:51:19.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:51:19.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:51:19.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:51:19.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:51:19.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-0/data/info.pb
03:51:19.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.48.126
03:51:19.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36597
03:51:19.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:51:19.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:51:19.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:51:19.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:51:19.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:51:19.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-0/logs
03:51:19.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:51:19.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:51:19.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:51:19.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:51:19.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:51:19.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:51:19.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:51:19.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 02:49:50 UTC on ccycloud.quasar-twsplx.root.comops.site
03:51:19.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4675617
03:51:19.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:51:19.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:19.010062 4648 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:51:19.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:19.011435 4648 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:51:19.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:19.032837 4672 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:51:19.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:19.656759 4661 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:51:19.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:19.657794 4661 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:51:20.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:20.439189 4679 raft_consensus.cc:482] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:51:20.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:20.440045 4679 raft_consensus.cc:504] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d31e45829e34488a283fca5aa119876" member_type: VOTER last_known_addr { host: "127.4.48.126" port: 43853 } } peers { permanent_uuid: "ead70800fa0e488da8e775fa5212d242" member_type: VOTER last_known_addr { host: "127.4.48.125" port: 35747 } } peers { permanent_uuid: "b08ed0a784ea44f7b4941b6eadaf4e13" member_type: VOTER last_known_addr { host: "127.4.48.124" port: 33343 } }
03:51:20.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:20.447971 4679 leader_election.cc:288] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3d31e45829e34488a283fca5aa119876 (127.4.48.126:43853), ead70800fa0e488da8e775fa5212d242 (127.4.48.125:35747)
03:51:20.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:20.462095 4598 leader_election.cc:334] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ead70800fa0e488da8e775fa5212d242 (127.4.48.125:35747): Network error: Client connection negotiation failed: client connection to 127.4.48.125:35747: connect: Connection refused (error 111)
03:51:20.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:20.462957 4595 leader_election.cc:334] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 3d31e45829e34488a283fca5aa119876 (127.4.48.126:43853): Network error: Client connection negotiation failed: client connection to 127.4.48.126:43853: connect: Connection refused (error 111)
03:51:20.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:20.463554 4595 leader_election.cc:302] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: b08ed0a784ea44f7b4941b6eadaf4e13; no voters: 3d31e45829e34488a283fca5aa119876, ead70800fa0e488da8e775fa5212d242
03:51:20.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:20.464375 4679 raft_consensus.cc:2726] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
03:51:20.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:19.037169 4675 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:51:20.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:19.033816 4673 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:51:20.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:20.531113 4674 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1490 milliseconds
03:51:20.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:20.531234 4648 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:51:20.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:20.532658 4648 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:51:20.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:20.535977 4648 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:51:20.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:20.537451 4648 hybrid_clock.cc:648] HybridClock initialized: now 1790481080537381 us; error 80 us; skew 500 ppm
03:51:20.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:20.538681 4648 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:51:20.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 03:51:20 dist-test-slave-c26w krb5kdc[4301](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790481080, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.48.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:51:20.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:20.551630 4648 init.cc:377] Logged in from keytab as kudu/127.4.48.126@KRBTEST.COM (short username kudu)
03:51:20.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:20.557857 4648 webserver.cc:460] Webserver started at http://127.4.48.126:36597/ using document root <none> and password file <none>
03:51:20.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:20.558780 4648 fs_manager.cc:359] Metadata directory not provided
03:51:20.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:20.559060 4648 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:51:20.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:20.568038 4648 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.008s sys 0.000s
03:51:20.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:20.572959 4688 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:51:20.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:20.574213 4648 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
03:51:20.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:20.574517 4648 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-0/data,/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-0/wal
03:51:20.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3d31e45829e34488a283fca5aa119876"
03:51:20.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 03:51:12 on dist-test-slave-c26w"
03:51:20.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:20.577072 4648 fs_report.cc:383] FS layout report
03:51:20.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:51:20.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-0/wal
03:51:20.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-0/wal
03:51:20.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-0/data/data
03:51:20.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:51:20.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:51:20.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:51:20.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:51:20.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:51:20.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:51:20.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:51:20.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:51:20.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:51:20.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:51:20.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:51:20.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:20.617950 4648 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:51:20.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:20.625630 4648 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:51:20.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:20.626109 4648 kserver.cc:163] Server-wide thread pool size limit: 3276
03:51:20.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:20.663283 4661 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:51:20.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:20.664027 4661 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:51:20.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:20.716143 4648 rpc_server.cc:307] RPC server started. Bound to: 127.4.48.126:43853
03:51:20.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:20.716259 4739 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.48.126:43853 every 8 connection(s)
03:51:20.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:20.719040 4648 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-0/data/info.pb
03:51:20.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:20.728040 4289 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskaXaVUt/build/tsan/bin/kudu as pid 4648
03:51:20.729 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:51:20.729 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.4.48.125:35747
03:51:20.730 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
03:51:20.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:20.732147 4740 sys_catalog.cc:263] Verifying existing consensus state
03:51:20.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:20.731997 4289 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskaXaVUt/build/tsan/bin/kudu
03:51:20.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskaXaVUt/build/tsan/bin/kudu
03:51:20.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-1/wal
03:51:20.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-1/data
03:51:20.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:51:20.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:51:20.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:51:20.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:51:20.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:51:20.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:51:20.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-1/logs
03:51:20.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-1/data/info.pb
03:51:20.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:51:20.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:51:20.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:51:20.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:51:20.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:51:20.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:51:20.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:51:20.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:51:20.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:51:20.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:51:20.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:51:20.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:51:20.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.48.125:35747
03:51:20.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.48.125
03:51:20.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44665
03:51:20.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/krb5kdc/kudu_127.4.48.125.keytab
03:51:20.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.48.125
03:51:20.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:51:20.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:51:20.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:51:20.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.48.84:43463
03:51:20.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:51:20.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:51:20.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:51:20.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:51:20.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.48.126:43853,127.4.48.125:35747,127.4.48.124:33343
03:51:20.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:51:20.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36253/jwks.json
03:51:20.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:51:20.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260927 03:51:20.733191 4740 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
03:51:20.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:20.733824 4648 master.cc:537] Master@127.4.48.126:43853 shutting down...
03:51:20.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:20.820007 4648 master.cc:552] Master@127.4.48.126:43853 shutdown complete.
03:51:20.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
03:51:21.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:21.102308 4742 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:51:21.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:21.103221 4742 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:51:21.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:21.104403 4742 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:51:21.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:21.160449 4742 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:51:21.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:21.160741 4742 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:51:21.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:21.161187 4742 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:51:21.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:21.161793 4742 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:51:21.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:21.162422 4742 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:51:21.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:21.163018 4742 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:51:21.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:21.163479 4742 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36253/jwks.json
03:51:21.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:21.222847 4742 master_runner.cc:386] Master server non-default flags:
03:51:21.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:51:21.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.48.84:43463
03:51:21.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:51:21.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:51:21.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-1/data
03:51:21.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-1/wal
03:51:21.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:51:21.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:51:21.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.48.126:43853,127.4.48.125:35747,127.4.48.124:33343
03:51:21.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/krb5kdc/kudu_127.4.48.125.keytab
03:51:21.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.48.125
03:51:21.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:51:21.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:51:21.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:51:21.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.48.125:35747
03:51:21.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:51:21.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:51:21.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:51:21.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36253/jwks.json
03:51:21.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:51:21.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:51:21.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:51:21.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:51:21.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:51:21.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-1/data/info.pb
03:51:21.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.48.125
03:51:21.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44665
03:51:21.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:51:21.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:51:21.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:51:21.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:51:21.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:51:21.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-1/logs
03:51:21.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:51:21.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:51:21.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:51:21.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:51:21.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:51:21.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:51:21.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:51:21.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 02:49:50 UTC on ccycloud.quasar-twsplx.root.comops.site
03:51:21.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4675617
03:51:21.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:51:21.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:21.224380 4742 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:51:21.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:21.225826 4742 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:51:21.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:21.248466 4749 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:51:21.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:21.670287 4661 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:51:21.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:21.670796 4661 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:51:22.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:21.250295 4742 server_base.cc:1034] running on GCE node
03:51:22.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:21.249977 4748 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:51:22.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:21.249831 4751 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:51:22.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:22.534099 4742 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:51:22.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:22.536908 4742 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:51:22.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:22.538606 4742 hybrid_clock.cc:648] HybridClock initialized: now 1790481082538561 us; error 49 us; skew 500 ppm
03:51:22.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:22.539706 4742 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:51:22.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 03:51:22 dist-test-slave-c26w krb5kdc[4301](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790481082, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.48.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:51:22.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:22.546216 4742 init.cc:377] Logged in from keytab as kudu/127.4.48.125@KRBTEST.COM (short username kudu)
03:51:22.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:22.551802 4742 webserver.cc:460] Webserver started at http://127.4.48.125:44665/ using document root <none> and password file <none>
03:51:22.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:22.552820 4742 fs_manager.cc:359] Metadata directory not provided
03:51:22.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:22.553098 4742 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:51:22.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:22.562399 4742 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.004s sys 0.002s
03:51:22.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:22.567706 4759 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:51:22.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:22.568970 4742 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
03:51:22.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:22.569365 4742 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-1/data,/tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-1/wal
03:51:22.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ead70800fa0e488da8e775fa5212d242"
03:51:22.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 03:51:14 on dist-test-slave-c26w"
03:51:22.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:22.571875 4742 fs_report.cc:383] FS layout report
03:51:22.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:51:22.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-1/wal
03:51:22.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-1/wal
03:51:22.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-1/data/data
03:51:22.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:51:22.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:51:22.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:51:22.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:51:22.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:51:22.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:51:22.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:51:22.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:51:22.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:51:22.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:51:22.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:51:22.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:22.624181 4742 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:51:22.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:22.631196 4742 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:51:22.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:22.631628 4742 kserver.cc:163] Server-wide thread pool size limit: 3276
03:51:22.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:22.673642 4661 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:51:22.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:22.673942 4661 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:51:22.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:22.702834 4742 rpc_server.cc:307] RPC server started. Bound to: 127.4.48.125:35747
03:51:22.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:22.703027 4810 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.48.125:35747 every 8 connection(s)
03:51:22.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:22.705672 4742 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-1/data/info.pb
03:51:22.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:22.708662 4289 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskaXaVUt/build/tsan/bin/kudu as pid 4742
03:51:22.709 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:51:22.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:22.719677 4811 sys_catalog.cc:263] Verifying existing consensus state
03:51:22.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260927 03:51:22.720752 4811 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
03:51:22.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:22.721271 4742 master.cc:537] Master@127.4.48.125:35747 shutting down...
03:51:22.777 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.4.48.124:33343(127.4.48.124:33343)] connecting to peer
03:51:22.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:22.823985 4742 master.cc:552] Master@127.4.48.125:35747 shutdown complete.
03:51:22.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskaXaVUt/test-tmp/mini-kudu-cluster6421972105673240358/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
03:51:22.857 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.4.48.126:43853(127.4.48.126:43853)] connecting to peer
03:51:22.859 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.4.48.125:35747(127.4.48.125:35747)] connecting to peer
03:51:22.909 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:22.909 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.48.126:43853(127.4.48.126:43853)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:22.911 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.48.126:43853: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:22.911 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.4.48.124:33343(127.4.48.124:33343)] Successfully connected to peer
03:51:22.973 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:22.973 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.48.125:35747(127.4.48.125:35747)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:22.974 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.48.125:35747: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:22.989 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
03:51:22.997 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.4.48.124
03:51:23.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 03:51:23 dist-test-slave-c26w krb5kdc[4301](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790481076, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.4.48.124@KRBTEST.COM
03:51:23.236 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xd800c307, L:/127.0.0.1:38562 - R:/127.4.48.124:33343] using SASL/GSSAPI
03:51:23.253 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747; will retry
03:51:23.278 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.48.126:43853(127.4.48.126:43853)] connecting to peer
03:51:23.280 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.48.125:35747(127.4.48.125:35747)] connecting to peer
03:51:23.281 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:23.282 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.48.126:43853(127.4.48.126:43853)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:23.284 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.48.126:43853: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:23.285 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:23.285 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.48.125:35747(127.4.48.125:35747)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:23.286 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.48.125:35747: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:23.287 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747; will retry
03:51:23.296 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.48.126:43853(127.4.48.126:43853)] connecting to peer
03:51:23.297 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.48.125:35747(127.4.48.125:35747)] connecting to peer
03:51:23.299 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:23.299 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.48.126:43853(127.4.48.126:43853)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:23.299 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.48.126:43853: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:23.301 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:23.301 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.48.125:35747(127.4.48.125:35747)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:23.302 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.48.125:35747: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:23.302 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747; will retry
03:51:23.315 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.48.126:43853(127.4.48.126:43853)] connecting to peer
03:51:23.318 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:23.318 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.48.126:43853(127.4.48.126:43853)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:23.318 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.48.125:35747(127.4.48.125:35747)] connecting to peer
03:51:23.320 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.48.126:43853: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:23.322 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:23.322 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.48.125:35747(127.4.48.125:35747)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:23.323 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.48.125:35747: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:23.323 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747; will retry
03:51:23.355 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.48.126:43853(127.4.48.126:43853)] connecting to peer
03:51:23.356 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.48.125:35747(127.4.48.125:35747)] connecting to peer
03:51:23.359 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:23.359 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.48.126:43853(127.4.48.126:43853)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:23.359 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.48.126:43853: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:23.360 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:23.360 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.48.125:35747(127.4.48.125:35747)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:23.361 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.48.125:35747: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:23.361 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747; will retry
03:51:23.395 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.48.126:43853(127.4.48.126:43853)] connecting to peer
03:51:23.397 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.48.125:35747(127.4.48.125:35747)] connecting to peer
03:51:23.399 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:23.399 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.48.126:43853(127.4.48.126:43853)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:23.400 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.48.126:43853: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:23.401 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:23.401 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.48.125:35747(127.4.48.125:35747)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:23.402 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.48.125:35747: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:23.403 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747; will retry
03:51:23.456 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.48.126:43853(127.4.48.126:43853)] connecting to peer
03:51:23.459 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:23.459 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.48.126:43853(127.4.48.126:43853)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:23.459 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.48.125:35747(127.4.48.125:35747)] connecting to peer
03:51:23.460 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.48.126:43853: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:23.461 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:23.462 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.48.125:35747(127.4.48.125:35747)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:23.462 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.48.125:35747: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:23.462 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747; will retry
03:51:23.556 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.48.126:43853(127.4.48.126:43853)] connecting to peer
03:51:23.558 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.48.125:35747(127.4.48.125:35747)] connecting to peer
03:51:23.558 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:23.559 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.48.126:43853(127.4.48.126:43853)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:23.559 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.48.126:43853: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:23.561 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:23.561 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.48.125:35747(127.4.48.125:35747)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:23.562 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.48.125:35747: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:23.562 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747; will retry
03:51:23.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:23.605232 4819 raft_consensus.cc:482] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:51:23.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:23.605587 4819 raft_consensus.cc:504] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d31e45829e34488a283fca5aa119876" member_type: VOTER last_known_addr { host: "127.4.48.126" port: 43853 } } peers { permanent_uuid: "ead70800fa0e488da8e775fa5212d242" member_type: VOTER last_known_addr { host: "127.4.48.125" port: 35747 } } peers { permanent_uuid: "b08ed0a784ea44f7b4941b6eadaf4e13" member_type: VOTER last_known_addr { host: "127.4.48.124" port: 33343 } }
03:51:23.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:23.607280 4819 leader_election.cc:288] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3d31e45829e34488a283fca5aa119876 (127.4.48.126:43853), ead70800fa0e488da8e775fa5212d242 (127.4.48.125:35747)
03:51:23.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:23.613931 4595 leader_election.cc:334] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 3d31e45829e34488a283fca5aa119876 (127.4.48.126:43853): Network error: Client connection negotiation failed: client connection to 127.4.48.126:43853: connect: Connection refused (error 111)
03:51:23.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:23.614461 4598 leader_election.cc:334] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ead70800fa0e488da8e775fa5212d242 (127.4.48.125:35747): Network error: Client connection negotiation failed: client connection to 127.4.48.125:35747: connect: Connection refused (error 111)
03:51:23.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:23.614965 4598 leader_election.cc:302] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: b08ed0a784ea44f7b4941b6eadaf4e13; no voters: 3d31e45829e34488a283fca5aa119876, ead70800fa0e488da8e775fa5212d242
03:51:23.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:23.615588 4819 raft_consensus.cc:2726] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
03:51:23.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:23.675674 4576 master_runner.cc:186] Error getting master registration for 127.4.48.126:43853: Network error: Client connection negotiation failed: client connection to 127.4.48.126:43853: connect: Connection refused (error 111),
03:51:23.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:23.676136 4576 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:51:23.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:23.677184 4661 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:51:23.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:23.677397 4661 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:51:23.696 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.48.126:43853(127.4.48.126:43853)] connecting to peer
03:51:23.698 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.48.125:35747(127.4.48.125:35747)] connecting to peer
03:51:23.699 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:23.700 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.48.126:43853(127.4.48.126:43853)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:23.702 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.48.126:43853: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:23.702 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:23.703 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.48.125:35747(127.4.48.125:35747)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:23.703 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.48.125:35747: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:23.704 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747; will retry
03:51:24.015 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.48.126:43853(127.4.48.126:43853)] connecting to peer
03:51:24.017 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.48.125:35747(127.4.48.125:35747)] connecting to peer
03:51:24.018 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:24.018 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.48.126:43853(127.4.48.126:43853)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:24.018 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:24.019 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.48.126:43853: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:24.019 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.48.125:35747(127.4.48.125:35747)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:24.020 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.48.125:35747: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:24.021 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747; will retry
03:51:24.476 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.48.126:43853(127.4.48.126:43853)] connecting to peer
03:51:24.477 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.48.125:35747(127.4.48.125:35747)] connecting to peer
03:51:24.479 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:24.479 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:24.479 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.48.125:35747(127.4.48.125:35747)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:24.479 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.48.126:43853(127.4.48.126:43853)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:24.479 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.48.125:35747: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:24.480 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.48.126:43853: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:24.481 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747; will retry
03:51:24.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:24.680385 4661 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:51:24.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:24.680637 4661 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:51:25.415 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.48.126:43853(127.4.48.126:43853)] connecting to peer
03:51:25.417 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:25.417 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.48.126:43853(127.4.48.126:43853)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:25.418 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.48.125:35747(127.4.48.125:35747)] connecting to peer
03:51:25.418 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.48.126:43853: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:25.420 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:25.420 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.48.125:35747(127.4.48.125:35747)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:25.420 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.48.125:35747: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:25.420 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747; will retry
03:51:25.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:25.683898 4661 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:51:25.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:25.684173 4661 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:51:26.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:26.686964 4661 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:51:26.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:26.687232 4661 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:51:26.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:26.965687 4828 raft_consensus.cc:482] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:51:26.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:26.966100 4828 raft_consensus.cc:504] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d31e45829e34488a283fca5aa119876" member_type: VOTER last_known_addr { host: "127.4.48.126" port: 43853 } } peers { permanent_uuid: "ead70800fa0e488da8e775fa5212d242" member_type: VOTER last_known_addr { host: "127.4.48.125" port: 35747 } } peers { permanent_uuid: "b08ed0a784ea44f7b4941b6eadaf4e13" member_type: VOTER last_known_addr { host: "127.4.48.124" port: 33343 } }
03:51:26.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:26.967877 4828 leader_election.cc:288] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3d31e45829e34488a283fca5aa119876 (127.4.48.126:43853), ead70800fa0e488da8e775fa5212d242 (127.4.48.125:35747)
03:51:26.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:26.969823 4595 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.48.126:43853: connect: Connection refused (error 111) [suppressed 5 similar messages]
03:51:26.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:26.972365 4598 leader_election.cc:334] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ead70800fa0e488da8e775fa5212d242 (127.4.48.125:35747): Network error: Client connection negotiation failed: client connection to 127.4.48.125:35747: connect: Connection refused (error 111)
03:51:26.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:26.973002 4595 leader_election.cc:334] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 3d31e45829e34488a283fca5aa119876 (127.4.48.126:43853): Network error: Client connection negotiation failed: client connection to 127.4.48.126:43853: connect: Connection refused (error 111)
03:51:26.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:26.973281 4595 leader_election.cc:302] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: b08ed0a784ea44f7b4941b6eadaf4e13; no voters: 3d31e45829e34488a283fca5aa119876, ead70800fa0e488da8e775fa5212d242
03:51:26.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:26.973820 4828 raft_consensus.cc:2726] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
03:51:27.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:27.690743 4661 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:51:27.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:27.691150 4661 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:51:28.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 03:51:28 dist-test-slave-c26w krb5kdc[4301](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790481088, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.48.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:51:28.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:28.372272 4590 init.cc:295] Successfully reacquired a new kerberos TGT
03:51:28.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:28.679612 4576 master_runner.cc:186] Error getting master registration for 127.4.48.126:43853: Network error: Client connection negotiation failed: client connection to 127.4.48.126:43853: connect: Connection refused (error 111),
03:51:28.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:28.680030 4576 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:51:28.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:28.694947 4661 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:51:28.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:28.695256 4661 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:51:29.436 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.48.126:43853(127.4.48.126:43853)] connecting to peer
03:51:29.437 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.48.125:35747(127.4.48.125:35747)] connecting to peer
03:51:29.438 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:29.438 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.48.126:43853(127.4.48.126:43853)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:29.438 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.48.126:43853: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:29.439 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:29.440 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.48.125:35747(127.4.48.125:35747)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:29.440 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.48.125:35747: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:29.441 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747; will retry
03:51:29.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:29.697980 4661 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:51:29.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:29.698244 4661 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:51:30.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:30.701179 4661 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:51:30.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:30.701422 4661 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:51:31.156 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.48.126:43853(127.4.48.126:43853)] connecting to peer
03:51:31.158 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:31.159 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.48.126:43853(127.4.48.126:43853)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:31.159 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.48.126:43853: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:31.160 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.48.125:35747(127.4.48.125:35747)] connecting to peer
03:51:31.163 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:31.163 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.48.125:35747(127.4.48.125:35747)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:31.163 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.48.125:35747: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:31.164 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747; will retry
03:51:31.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:31.704312 4661 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:51:31.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:31.704563 4661 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:51:32.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:32.707464 4661 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:51:32.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:32.707712 4661 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:51:33.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:33.683005 4576 master_runner.cc:186] Error getting master registration for 127.4.48.126:43853: Network error: Client connection negotiation failed: client connection to 127.4.48.126:43853: connect: Connection refused (error 111),
03:51:33.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:33.683346 4576 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:51:33.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:33.710557 4661 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:51:33.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:33.710850 4661 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:51:33.896 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.48.126:43853(127.4.48.126:43853)] connecting to peer
03:51:33.900 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:33.901 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.48.125:35747(127.4.48.125:35747)] connecting to peer
03:51:33.901 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.48.126:43853(127.4.48.126:43853)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:33.901 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.48.126:43853: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:33.902 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:33.903 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.48.125:35747(127.4.48.125:35747)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:33.903 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.48.125:35747: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:33.903 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747; will retry
03:51:34.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:34.104975 4841 raft_consensus.cc:482] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:51:34.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:34.105419 4841 raft_consensus.cc:504] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d31e45829e34488a283fca5aa119876" member_type: VOTER last_known_addr { host: "127.4.48.126" port: 43853 } } peers { permanent_uuid: "ead70800fa0e488da8e775fa5212d242" member_type: VOTER last_known_addr { host: "127.4.48.125" port: 35747 } } peers { permanent_uuid: "b08ed0a784ea44f7b4941b6eadaf4e13" member_type: VOTER last_known_addr { host: "127.4.48.124" port: 33343 } }
03:51:34.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:34.107508 4841 leader_election.cc:288] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3d31e45829e34488a283fca5aa119876 (127.4.48.126:43853), ead70800fa0e488da8e775fa5212d242 (127.4.48.125:35747)
03:51:34.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:34.108749 4595 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.48.126:43853: connect: Connection refused (error 111) [suppressed 1 similar messages]
03:51:34.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:34.111946 4595 leader_election.cc:334] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 3d31e45829e34488a283fca5aa119876 (127.4.48.126:43853): Network error: Client connection negotiation failed: client connection to 127.4.48.126:43853: connect: Connection refused (error 111)
03:51:34.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:34.112507 4598 leader_election.cc:334] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ead70800fa0e488da8e775fa5212d242 (127.4.48.125:35747): Network error: Client connection negotiation failed: client connection to 127.4.48.125:35747: connect: Connection refused (error 111)
03:51:34.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:34.112790 4598 leader_election.cc:302] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: b08ed0a784ea44f7b4941b6eadaf4e13; no voters: 3d31e45829e34488a283fca5aa119876, ead70800fa0e488da8e775fa5212d242
03:51:34.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:34.113356 4841 raft_consensus.cc:2726] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
03:51:34.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:34.713673 4661 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:51:34.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:34.713969 4661 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:51:35.616 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.48.126:43853(127.4.48.126:43853)] connecting to peer
03:51:35.617 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.48.125:35747(127.4.48.125:35747)] connecting to peer
03:51:35.619 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:35.619 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:35.619 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.48.126:43853(127.4.48.126:43853)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:35.619 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.48.125:35747(127.4.48.125:35747)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:35.620 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.48.126:43853: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:35.620 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.48.125:35747: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:35.621 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747; will retry
03:51:35.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:35.716931 4661 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:51:35.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:35.717182 4661 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:51:36.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:36.719919 4661 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:51:36.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:36.720225 4661 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:51:37.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:37.723771 4661 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:51:37.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:37.724191 4661 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:51:38.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 03:51:38 dist-test-slave-c26w krb5kdc[4301](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790481098, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.48.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:51:38.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:38.374600 4590 init.cc:295] Successfully reacquired a new kerberos TGT
03:51:38.656 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.48.126:43853(127.4.48.126:43853)] connecting to peer
03:51:38.658 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.48.125:35747(127.4.48.125:35747)] connecting to peer
03:51:38.658 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:38.659 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.48.126:43853(127.4.48.126:43853)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:38.659 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.48.126:43853: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:38.660 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:38.661 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.48.125:35747(127.4.48.125:35747)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:38.661 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.48.125:35747: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:38.661 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747; will retry
03:51:38.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:38.686472 4576 master_runner.cc:186] Error getting master registration for 127.4.48.126:43853: Network error: Client connection negotiation failed: client connection to 127.4.48.126:43853: connect: Connection refused (error 111),
03:51:38.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:38.686930 4576 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:51:38.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:38.727263 4661 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:51:38.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:38.727536 4661 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:51:39.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:39.730556 4661 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:51:39.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:39.730818 4661 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:51:40.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:40.733644 4661 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:51:40.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:40.733969 4661 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:51:41.355 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.48.126:43853(127.4.48.126:43853)] connecting to peer
03:51:41.357 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.48.125:35747(127.4.48.125:35747)] connecting to peer
03:51:41.358 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:41.358 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.48.126:43853(127.4.48.126:43853)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:41.358 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.48.126:43853: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:41.360 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:41.360 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.48.125:35747(127.4.48.125:35747)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:41.360 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.48.125:35747: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:41.360 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747; will retry
03:51:41.736 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.48.126:43853(127.4.48.126:43853)] connecting to peer
03:51:41.737 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.48.125:35747(127.4.48.125:35747)] connecting to peer
03:51:41.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:41.738040 4661 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:51:41.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:41.738535 4661 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:51:41.739 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:41.740 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.48.126:43853(127.4.48.126:43853)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:41.740 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.48.126:43853: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:41.741 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:41.741 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.48.125:35747(127.4.48.125:35747)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:41.741 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.48.125:35747: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:41.741 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747; will retry
03:51:42.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:42.741432 4661 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:51:42.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:42.741747 4661 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:51:42.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:42.999361 4852 raft_consensus.cc:482] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:51:43.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:42.999814 4852 raft_consensus.cc:504] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d31e45829e34488a283fca5aa119876" member_type: VOTER last_known_addr { host: "127.4.48.126" port: 43853 } } peers { permanent_uuid: "ead70800fa0e488da8e775fa5212d242" member_type: VOTER last_known_addr { host: "127.4.48.125" port: 35747 } } peers { permanent_uuid: "b08ed0a784ea44f7b4941b6eadaf4e13" member_type: VOTER last_known_addr { host: "127.4.48.124" port: 33343 } }
03:51:43.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:43.001529 4852 leader_election.cc:288] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3d31e45829e34488a283fca5aa119876 (127.4.48.126:43853), ead70800fa0e488da8e775fa5212d242 (127.4.48.125:35747)
03:51:43.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:43.003925 4595 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.48.126:43853: connect: Connection refused (error 111) [suppressed 1 similar messages]
03:51:43.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:43.006997 4598 leader_election.cc:334] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ead70800fa0e488da8e775fa5212d242 (127.4.48.125:35747): Network error: Client connection negotiation failed: client connection to 127.4.48.125:35747: connect: Connection refused (error 111)
03:51:43.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:43.007481 4595 leader_election.cc:334] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 3d31e45829e34488a283fca5aa119876 (127.4.48.126:43853): Network error: Client connection negotiation failed: client connection to 127.4.48.126:43853: connect: Connection refused (error 111)
03:51:43.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:43.007769 4595 leader_election.cc:302] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: b08ed0a784ea44f7b4941b6eadaf4e13; no voters: 3d31e45829e34488a283fca5aa119876, ead70800fa0e488da8e775fa5212d242
03:51:43.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:43.008500 4852 raft_consensus.cc:2726] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
03:51:43.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:43.689527 4576 master_runner.cc:186] Error getting master registration for 127.4.48.126:43853: Network error: Client connection negotiation failed: client connection to 127.4.48.126:43853: connect: Connection refused (error 111),
03:51:43.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:43.689899 4576 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:51:43.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:43.744676 4661 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:51:43.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:43.744930 4661 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:51:44.356 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.48.126:43853(127.4.48.126:43853)] connecting to peer
03:51:44.358 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.48.125:35747(127.4.48.125:35747)] connecting to peer
03:51:44.358 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:44.358 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.48.126:43853(127.4.48.126:43853)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:44.359 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.48.126:43853: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:44.360 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:44.360 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.48.125:35747(127.4.48.125:35747)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:44.360 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.48.125:35747: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:44.360 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747; will retry
03:51:44.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:44.747687 4661 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:51:44.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:44.747933 4661 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:51:45.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:45.750963 4661 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:51:45.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:45.751253 4661 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:51:46.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:46.754063 4661 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:51:46.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:46.754344 4661 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:51:47.240 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.48.126:43853(127.4.48.126:43853)] connecting to peer
03:51:47.242 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:47.242 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.48.126:43853(127.4.48.126:43853)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:47.242 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.48.125:35747(127.4.48.125:35747)] connecting to peer
03:51:47.242 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.48.126:43853: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:47.243 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:47.243 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.48.125:35747(127.4.48.125:35747)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:47.243 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.48.125:35747: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:47.243 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747; will retry
03:51:47.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:47.758472 4661 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:51:47.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:47.758837 4661 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:51:48.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 03:51:48 dist-test-slave-c26w krb5kdc[4301](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790481108, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.48.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:51:48.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:48.377204 4590 init.cc:295] Successfully reacquired a new kerberos TGT
03:51:48.575 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.48.126:43853(127.4.48.126:43853)] connecting to peer
03:51:48.577 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.48.125:35747(127.4.48.125:35747)] connecting to peer
03:51:48.577 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:48.577 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.48.126:43853(127.4.48.126:43853)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:48.577 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.48.126:43853: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:48.578 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:48.578 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.48.125:35747(127.4.48.125:35747)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:48.579 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.48.125:35747: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:48.580 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747; will retry
03:51:48.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:48.692992 4576 master_runner.cc:186] Error getting master registration for 127.4.48.126:43853: Network error: Client connection negotiation failed: client connection to 127.4.48.126:43853: connect: Connection refused (error 111),
03:51:48.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:48.693378 4576 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:51:48.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:48.761763 4661 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:51:48.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:48.762033 4661 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:51:49.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:49.764804 4661 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b08ed0a784ea44f7b4941b6eadaf4e13: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:51:49.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:51:49.765079 4661 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:51:50.256 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.48.126:43853(127.4.48.126:43853)] connecting to peer
03:51:50.257 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.48.125:35747(127.4.48.125:35747)] connecting to peer
03:51:50.257 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:50.257 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.48.126:43853(127.4.48.126:43853)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:50.257 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.48.126:43853: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853
03:51:50.258 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:50.258 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.48.125:35747(127.4.48.125:35747)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:50.258 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.48.125:35747: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
03:51:50.259 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747; will retry
03:51:50.264 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=23, TimeoutTracker(timeout=30000, elapsed=27546), Traces: [0ms] refreshing cache from master, [51ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.124:33343, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.126:43853, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.125:35747, [192ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.126:43853: Network error: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853, [256ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.125:35747: Network error: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [531ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.124:33343: OK, [536ms] delaying RPC due to: Service unavailable: Master config (127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [557ms] refreshing cache from master, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.124:33343, [560ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.126:43853, [562ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.125:35747, [563ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.124:33343: OK, [564ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.126:43853: Network error: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853, [567ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.125:35747: Network error: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [569ms] delaying RPC due to: Service unavailable: Master config (127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [577ms] refreshing cache from master, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.124:33343, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.126:43853, [579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.125:35747, [581ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.126:43853: Network error: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853, [583ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.124:33343: OK, [583ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.125:35747: Network error: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [584ms] delaying RPC due to: Service unavailable: Master config (127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [596ms] refreshing cache from master, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.124:33343, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.126:43853, [600ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.125:35747, [600ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.126:43853: Network error: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853, [601ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.124:33343: OK, [604ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.125:35747: Network error: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [605ms] delaying RPC due to: Service unavailable: Master config (127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [636ms] refreshing cache from master, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.124:33343, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.126:43853, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.125:35747, [641ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.126:43853: Network error: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853, [642ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.124:33343: OK, [643ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.125:35747: Network error: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [644ms] delaying RPC due to: Service unavailable: Master config (127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [677ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.124:33343, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.126:43853, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.125:35747, [681ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.126:43853: Network error: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853, [683ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.124:33343: OK, [683ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.125:35747: Network error: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [685ms] delaying RPC due to: Service unavailable: Master config (127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [737ms] refreshing cache from master, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.124:33343, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.126:43853, [739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.125:35747, [741ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.126:43853: Network error: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853, [742ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.124:33343: OK, [744ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.125:35747: Network error: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [745ms] delaying RPC due to: Service unavailable: Master config (127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [837ms] refreshing cache from master, [837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.124:33343, [838ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.126:43853, [839ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.125:35747, [841ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.126:43853: Network error: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853, [842ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.124:33343: OK, [843ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.125:35747: Network error: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [845ms] delaying RPC due to: Service unavailable: Master config (127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [977ms] refreshing cache from master, [977ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.124:33343, [978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.126:43853, [979ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.125:35747, [983ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.126:43853: Network error: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853, [984ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.124:33343: OK, [985ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.125:35747: Network error: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [986ms] delaying RPC due to: Service unavailable: Master config (127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [1297ms] refreshing cache from master, [1297ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.124:33343, [1297ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.126:43853, [1299ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.125:35747, [1301ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.126:43853: Network error: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853, [1301ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.125:35747: Network error: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [1303ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.124:33343: OK, [1304ms] delaying RPC due to: Service unavailable: Master config (127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [1757ms] refreshing cache from master, [1757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.124:33343, [1758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.126:43853, [1758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.125:35747, [1761ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.126:43853: Network error: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853, [1761ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.125:35747: Network error: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [1762ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.124:33343: OK, [1763ms] delaying RPC due to: Service unavailable: Master config (127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853, [2696ms] refreshing cache from master, [2697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.124:33343, [2697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.126:43853, [2699ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.125:35747, [2700ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.126:43853: Network error: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853, [2701ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.124:33343: OK, [2702ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.125:35747: Network error: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [2702ms] delaying RPC due to: Service unavailable: Master config (127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [6717ms] refreshing cache from master, [6717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.124:33343, [6718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.126:43853, [6719ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.125:35747, [6720ms] too many traces: truncated at 100 traces, deferred=Deferred@1727502134(state=PENDING, result=null, callback=(continuation of Deferred@1998747194 after retry RPC after error@190042322) -> (continuation of Deferred@330704161 after retry RPC after error@1235377734) -> (continuation of Deferred@760574702 after retry RPC after error@940962518) -> (continuation of Deferred@447236056 after retry RPC after error@1092844197) -> (continuation of Deferred@1206362733 after retry RPC after error@1358134021) -> (continuation of Deferred@138240068 after retry RPC after error@1376896436) -> (continuation of Deferred@673177937 after retry RPC after error@1677696004) -> (continuation of Deferred@2359167 after retry RPC after error@1547937450) -> (continuation of Deferred@193057050 after retry RPC after error@344024018) -> (continuation of Deferred@189181498 after retry RPC after error@1248935694) -> (continuation of Deferred@1341992539 after retry RPC after error@1358777221) -> (continuation of Deferred@1853102302 after retry RPC after error@767480657) -> (continuation of Deferred@194490187 after retry RPC after error@822901983) -> (continuation of Deferred@2138058952 after retry RPC after error@613367852) -> (continuation of Deferred@307514881 after retry RPC after error@1080832528) -> (continuation of Deferred@1046789285 after retry RPC after error@1614021989) -> (continuation of Deferred@1344301860 after retry RPC after error@1912418178) -> (continuation of Deferred@1793537164 after retry RPC after error@708320152) -> (continuation of Deferred@2031234224 after retry RPC after error@1628151087) -> (continuation of Deferred@849057494 after retry RPC after error@1486204870) -> (continuation of Deferred@805990344 after retry RPC after error@223702531) -> (continuation of Deferred@1670804569 after retry RPC after error@369626226), errback=(continuation of Deferred@1998747194 after retry RPC after error@190042322) -> (continuation of Deferred@330704161 after retry RPC after error@1235377734) -> (continuation of Deferred@760574702 after retry RPC after error@940962518) -> (continuation of Deferred@447236056 after retry RPC after error@1092844197) -> (continuation of Deferred@1206362733 after retry RPC after error@1358134021) -> (continuation of Deferred@138240068 after retry RPC after error@1376896436) -> (continuation of Deferred@673177937 after retry RPC after error@1677696004) -> (continuation of Deferred@2359167 after retry RPC after error@1547937450) -> (continuation of Deferred@193057050 after retry RPC after error@344024018) -> (continuation of Deferred@189181498 after retry RPC after error@1248935694) -> (continuation of Deferred@1341992539 after retry RPC after error@1358777221) -> (continuation of Deferred@1853102302 after retry RPC after error@767480657) -> (continuation of Deferred@194490187 after retry RPC after error@822901983) -> (continuation of Deferred@2138058952 after retry RPC after error@613367852) -> (continuation of Deferred@307514881 after retry RPC after error@1080832528) -> (continuation of Deferred@1046789285 after retry RPC after error@1614021989) -> (continuation of Deferred@1344301860 after retry RPC after error@1912418178) -> (continuation of Deferred@1793537164 after retry RPC after error@708320152) -> (continuation of Deferred@2031234224 after retry RPC after error@1628151087) -> (continuation of Deferred@849057494 after retry RPC after error@1486204870) -> (continuation of Deferred@805990344 after retry RPC after error@223702531) -> (continuation of Deferred@1670804569 after retry RPC after error@369626226)))
03:51:50.269 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.4.48.125:35747(127.4.48.125:35747)] disconnecting while in state TERMINATED
03:51:50.270 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.4.48.124:33343(127.4.48.124:33343)] disconnecting while in state READY
03:51:50.270 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.4.48.126:43853(127.4.48.126:43853)] disconnecting while in state TERMINATED
03:51:50.275 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
03:51:50.276 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
03:51:50.278 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.4.48.124:33343(127.4.48.124:33343)] handling channelInactive
03:51:50.278 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.48.124:33343(127.4.48.124:33343)] cleaning up while in state READY due to: connection closed
03:51:50.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:50.283701 4289 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskaXaVUt/build/tsan/bin/kudu with pid 4576
03:51:51.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T03:51:51Z chronyd exiting
03:51:52.547 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
03:51:52.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:52.548523 3563 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskaXaVUt/build/tsan/bin/kudu with pid 3845
03:51:52.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:52.574991 3563 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskaXaVUt/build/tsan/bin/kudu with pid 4004
03:51:52.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:52.602020 3563 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskaXaVUt/build/tsan/bin/kudu with pid 4148
03:51:52.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:52.627732 3563 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskaXaVUt/build/tsan/bin/kudu with pid 3575
03:51:52.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:52.658046 3563 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskaXaVUt/build/tsan/bin/kudu with pid 3645
03:51:52.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:51:52.687600 3563 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskaXaVUt/build/tsan/bin/kudu with pid 3720
03:51:53.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T03:51:53Z chronyd exiting
03:51:54.872 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=23, TimeoutTracker(timeout=30000, elapsed=27546), Traces: [0ms] refreshing cache from master, [51ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.124:33343, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.126:43853, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.125:35747, [192ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.126:43853: Network error: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853, [256ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.125:35747: Network error: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [531ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.124:33343: OK, [536ms] delaying RPC due to: Service unavailable: Master config (127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [557ms] refreshing cache from master, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.124:33343, [560ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.126:43853, [562ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.125:35747, [563ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.124:33343: OK, [564ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.126:43853: Network error: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853, [567ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.125:35747: Network error: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [569ms] delaying RPC due to: Service unavailable: Master config (127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [577ms] refreshing cache from master, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.124:33343, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.126:43853, [579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.125:35747, [581ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.126:43853: Network error: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853, [583ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.124:33343: OK, [583ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.125:35747: Network error: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [584ms] delaying RPC due to: Service unavailable: Master config (127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [596ms] refreshing cache from master, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.124:33343, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.126:43853, [600ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.125:35747, [600ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.126:43853: Network error: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853, [601ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.124:33343: OK, [604ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.125:35747: Network error: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [605ms] delaying RPC due to: Service unavailable: Master config (127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [636ms] refreshing cache from master, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.124:33343, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.126:43853, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.125:35747, [641ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.126:43853: Network error: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853, [642ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.124:33343: OK, [643ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.125:35747: Network error: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [644ms] delaying RPC due to: Service unavailable: Master config (127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [677ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.124:33343, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.126:43853, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.125:35747, [681ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.126:43853: Network error: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853, [683ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.124:33343: OK, [683ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.125:35747: Network error: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [685ms] delaying RPC due to: Service unavailable: Master config (127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [737ms] refreshing cache from master, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.124:33343, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.126:43853, [739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.125:35747, [741ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.126:43853: Network error: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853, [742ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.124:33343: OK, [744ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.125:35747: Network error: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [745ms] delaying RPC due to: Service unavailable: Master config (127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [837ms] refreshing cache from master, [837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.124:33343, [838ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.126:43853, [839ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.125:35747, [841ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.126:43853: Network error: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853, [842ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.124:33343: OK, [843ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.125:35747: Network error: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [845ms] delaying RPC due to: Service unavailable: Master config (127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [977ms] refreshing cache from master, [977ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.124:33343, [978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.126:43853, [979ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.125:35747, [983ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.126:43853: Network error: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853, [984ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.124:33343: OK, [985ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.125:35747: Network error: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [986ms] delaying RPC due to: Service unavailable: Master config (127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [1297ms] refreshing cache from master, [1297ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.124:33343, [1297ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.126:43853, [1299ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.125:35747, [1301ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.126:43853: Network error: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853, [1301ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.125:35747: Network error: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [1303ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.124:33343: OK, [1304ms] delaying RPC due to: Service unavailable: Master config (127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [1757ms] refreshing cache from master, [1757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.124:33343, [1758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.126:43853, [1758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.125:35747, [1761ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.126:43853: Network error: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853, [1761ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.125:35747: Network error: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [1762ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.124:33343: OK, [1763ms] delaying RPC due to: Service unavailable: Master config (127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853, [2696ms] refreshing cache from master, [2697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.124:33343, [2697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.126:43853, [2699ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.125:35747, [2700ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.126:43853: Network error: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853, [2701ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.124:33343: OK, [2702ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.125:35747: Network error: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [2702ms] delaying RPC due to: Service unavailable: Master config (127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [6717ms] refreshing cache from master, [6717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.124:33343, [6718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.126:43853, [6719ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.125:35747, [6720ms] too many traces: truncated at 100 traces, deferred=Deferred@1727502134(state=PENDING, result=null, callback=(continuation of Deferred@1998747194 after retry RPC after error@190042322) -> (continuation of Deferred@330704161 after retry RPC after error@1235377734) -> (continuation of Deferred@760574702 after retry RPC after error@940962518) -> (continuation of Deferred@447236056 after retry RPC after error@1092844197) -> (continuation of Deferred@1206362733 after retry RPC after error@1358134021) -> (continuation of Deferred@138240068 after retry RPC after error@1376896436) -> (continuation of Deferred@673177937 after retry RPC after error@1677696004) -> (continuation of Deferred@2359167 after retry RPC after error@1547937450) -> (continuation of Deferred@193057050 after retry RPC after error@344024018) -> (continuation of Deferred@189181498 after retry RPC after error@1248935694) -> (continuation of Deferred@1341992539 after retry RPC after error@1358777221) -> (continuation of Deferred@1853102302 after retry RPC after error@767480657) -> (continuation of Deferred@194490187 after retry RPC after error@822901983) -> (continuation of Deferred@2138058952 after retry RPC after error@613367852) -> (continuation of Deferred@307514881 after retry RPC after error@1080832528) -> (continuation of Deferred@1046789285 after retry RPC after error@1614021989) -> (continuation of Deferred@1344301860 after retry RPC after error@1912418178) -> (continuation of Deferred@1793537164 after retry RPC after error@708320152) -> (continuation of Deferred@2031234224 after retry RPC after error@1628151087) -> (continuation of Deferred@849057494 after retry RPC after error@1486204870) -> (continuation of Deferred@805990344 after retry RPC after error@223702531) -> (continuation of Deferred@1670804569 after retry RPC after error@369626226), errback=(continuation of Deferred@1998747194 after retry RPC after error@190042322) -> (continuation of Deferred@330704161 after retry RPC after error@1235377734) -> (continuation of Deferred@760574702 after retry RPC after error@940962518) -> (continuation of Deferred@447236056 after retry RPC after error@1092844197) -> (continuation of Deferred@1206362733 after retry RPC after error@1358134021) -> (continuation of Deferred@138240068 after retry RPC after error@1376896436) -> (continuation of Deferred@673177937 after retry RPC after error@1677696004) -> (continuation of Deferred@2359167 after retry RPC after error@1547937450) -> (continuation of Deferred@193057050 after retry RPC after error@344024018) -> (continuation of Deferred@189181498 after retry RPC after error@1248935694) -> (continuation of Deferred@1341992539 after retry RPC after error@1358777221) -> (continuation of Deferred@1853102302 after retry RPC after error@767480657) -> (continuation of Deferred@194490187 after retry RPC after error@822901983) -> (continuation of Deferred@2138058952 after retry RPC after error@613367852) -> (continuation of Deferred@307514881 after retry RPC after error@1080832528) -> (continuation of Deferred@1046789285 after retry RPC after error@1614021989) -> (continuation of Deferred@1344301860 after retry RPC after error@1912418178) -> (continuation of Deferred@1793537164 after retry RPC after error@708320152) -> (continuation of Deferred@2031234224 after retry RPC after error@1628151087) -> (continuation of Deferred@849057494 after retry RPC after error@1486204870) -> (continuation of Deferred@805990344 after retry RPC after error@223702531) -> (continuation of Deferred@1670804569 after retry RPC after error@369626226)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.access$200(DefaultPromise.java:35) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise$1.run(DefaultPromise.java:503) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.4.48.125:35747
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=23, TimeoutTracker(timeout=30000, elapsed=27546), Traces: [0ms] refreshing cache from master, [51ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.124:33343, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.126:43853, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.125:35747, [192ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.126:43853: Network error: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853, [256ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.125:35747: Network error: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [531ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.124:33343: OK, [536ms] delaying RPC due to: Service unavailable: Master config (127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [557ms] refreshing cache from master, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.124:33343, [560ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.126:43853, [562ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.125:35747, [563ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.124:33343: OK, [564ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.126:43853: Network error: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853, [567ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.125:35747: Network error: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [569ms] delaying RPC due to: Service unavailable: Master config (127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [577ms] refreshing cache from master, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.124:33343, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.126:43853, [579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.125:35747, [581ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.126:43853: Network error: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853, [583ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.124:33343: OK, [583ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.125:35747: Network error: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [584ms] delaying RPC due to: Service unavailable: Master config (127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [596ms] refreshing cache from master, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.124:33343, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.126:43853, [600ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.125:35747, [600ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.126:43853: Network error: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853, [601ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.124:33343: OK, [604ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.125:35747: Network error: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [605ms] delaying RPC due to: Service unavailable: Master config (127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [636ms] refreshing cache from master, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.124:33343, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.126:43853, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.125:35747, [641ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.126:43853: Network error: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853, [642ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.124:33343: OK, [643ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.125:35747: Network error: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [644ms] delaying RPC due to: Service unavailable: Master config (127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [677ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.124:33343, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.126:43853, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.125:35747, [681ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.126:43853: Network error: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853, [683ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.124:33343: OK, [683ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.125:35747: Network error: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [685ms] delaying RPC due to: Service unavailable: Master config (127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [737ms] refreshing cache from master, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.124:33343, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.126:43853, [739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.125:35747, [741ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.126:43853: Network error: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853, [742ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.124:33343: OK, [744ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.125:35747: Network error: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [745ms] delaying RPC due to: Service unavailable: Master config (127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [837ms] refreshing cache from master, [837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.124:33343, [838ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.126:43853, [839ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.125:35747, [841ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.126:43853: Network error: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853, [842ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.124:33343: OK, [843ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.125:35747: Network error: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [845ms] delaying RPC due to: Service unavailable: Master config (127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [977ms] refreshing cache from master, [977ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.124:33343, [978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.126:43853, [979ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.125:35747, [983ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.126:43853: Network error: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853, [984ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.124:33343: OK, [985ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.125:35747: Network error: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [986ms] delaying RPC due to: Service unavailable: Master config (127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [1297ms] refreshing cache from master, [1297ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.124:33343, [1297ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.126:43853, [1299ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.125:35747, [1301ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.126:43853: Network error: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853, [1301ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.125:35747: Network error: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [1303ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.124:33343: OK, [1304ms] delaying RPC due to: Service unavailable: Master config (127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [1757ms] refreshing cache from master, [1757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.124:33343, [1758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.126:43853, [1758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.125:35747, [1761ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.126:43853: Network error: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853, [1761ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.125:35747: Network error: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [1762ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.124:33343: OK, [1763ms] delaying RPC due to: Service unavailable: Master config (127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853, [2696ms] refreshing cache from master, [2697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.124:33343, [2697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.126:43853, [2699ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.125:35747, [2700ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.126:43853: Network error: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853, [2701ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.124:33343: OK, [2702ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.125:35747: Network error: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [2702ms] delaying RPC due to: Service unavailable: Master config (127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [6717ms] refreshing cache from master, [6717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.124:33343, [6718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.126:43853, [6719ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.125:35747, [6720ms] too many traces: truncated at 100 traces, deferred=Deferred@1727502134(state=PENDING, result=null, callback=(continuation of Deferred@1998747194 after retry RPC after error@190042322) -> (continuation of Deferred@330704161 after retry RPC after error@1235377734) -> (continuation of Deferred@760574702 after retry RPC after error@940962518) -> (continuation of Deferred@447236056 after retry RPC after error@1092844197) -> (continuation of Deferred@1206362733 after retry RPC after error@1358134021) -> (continuation of Deferred@138240068 after retry RPC after error@1376896436) -> (continuation of Deferred@673177937 after retry RPC after error@1677696004) -> (continuation of Deferred@2359167 after retry RPC after error@1547937450) -> (continuation of Deferred@193057050 after retry RPC after error@344024018) -> (continuation of Deferred@189181498 after retry RPC after error@1248935694) -> (continuation of Deferred@1341992539 after retry RPC after error@1358777221) -> (continuation of Deferred@1853102302 after retry RPC after error@767480657) -> (continuation of Deferred@194490187 after retry RPC after error@822901983) -> (continuation of Deferred@2138058952 after retry RPC after error@613367852) -> (continuation of Deferred@307514881 after retry RPC after error@1080832528) -> (continuation of Deferred@1046789285 after retry RPC after error@1614021989) -> (continuation of Deferred@1344301860 after retry RPC after error@1912418178) -> (continuation of Deferred@1793537164 after retry RPC after error@708320152) -> (continuation of Deferred@2031234224 after retry RPC after error@1628151087) -> (continuation of Deferred@849057494 after retry RPC after error@1486204870) -> (continuation of Deferred@805990344 after retry RPC after error@223702531) -> (continuation of Deferred@1670804569 after retry RPC after error@369626226), errback=(continuation of Deferred@1998747194 after retry RPC after error@190042322) -> (continuation of Deferred@330704161 after retry RPC after error@1235377734) -> (continuation of Deferred@760574702 after retry RPC after error@940962518) -> (continuation of Deferred@447236056 after retry RPC after error@1092844197) -> (continuation of Deferred@1206362733 after retry RPC after error@1358134021) -> (continuation of Deferred@138240068 after retry RPC after error@1376896436) -> (continuation of Deferred@673177937 after retry RPC after error@1677696004) -> (continuation of Deferred@2359167 after retry RPC after error@1547937450) -> (continuation of Deferred@193057050 after retry RPC after error@344024018) -> (continuation of Deferred@189181498 after retry RPC after error@1248935694) -> (continuation of Deferred@1341992539 after retry RPC after error@1358777221) -> (continuation of Deferred@1853102302 after retry RPC after error@767480657) -> (continuation of Deferred@194490187 after retry RPC after error@822901983) -> (continuation of Deferred@2138058952 after retry RPC after error@613367852) -> (continuation of Deferred@307514881 after retry RPC after error@1080832528) -> (continuation of Deferred@1046789285 after retry RPC after error@1614021989) -> (continuation of Deferred@1344301860 after retry RPC after error@1912418178) -> (continuation of Deferred@1793537164 after retry RPC after error@708320152) -> (continuation of Deferred@2031234224 after retry RPC after error@1628151087) -> (continuation of Deferred@849057494 after retry RPC after error@1486204870) -> (continuation of Deferred@805990344 after retry RPC after error@223702531) -> (continuation of Deferred@1670804569 after retry RPC after error@369626226)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557)
at io.netty.util.concurrent.DefaultPromise.access$200(DefaultPromise.java:35)
at io.netty.util.concurrent.DefaultPromise$1.run(DefaultPromise.java:503)
at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174)
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167)
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569)
... 5 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.4.48.125:35747
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
03:51:55.722 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
03:51:55.723 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=23, TimeoutTracker(timeout=30000, elapsed=27546), Traces: [0ms] refreshing cache from master, [51ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.124:33343, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.126:43853, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.125:35747, [192ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.126:43853: Network error: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853, [256ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.125:35747: Network error: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [531ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.124:33343: OK, [536ms] delaying RPC due to: Service unavailable: Master config (127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [557ms] refreshing cache from master, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.124:33343, [560ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.126:43853, [562ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.125:35747, [563ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.124:33343: OK, [564ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.126:43853: Network error: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853, [567ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.125:35747: Network error: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [569ms] delaying RPC due to: Service unavailable: Master config (127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [577ms] refreshing cache from master, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.124:33343, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.126:43853, [579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.125:35747, [581ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.126:43853: Network error: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853, [583ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.124:33343: OK, [583ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.125:35747: Network error: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [584ms] delaying RPC due to: Service unavailable: Master config (127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [596ms] refreshing cache from master, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.124:33343, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.126:43853, [600ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.125:35747, [600ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.126:43853: Network error: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853, [601ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.124:33343: OK, [604ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.125:35747: Network error: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [605ms] delaying RPC due to: Service unavailable: Master config (127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [636ms] refreshing cache from master, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.124:33343, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.126:43853, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.125:35747, [641ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.126:43853: Network error: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853, [642ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.124:33343: OK, [643ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.125:35747: Network error: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [644ms] delaying RPC due to: Service unavailable: Master config (127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [677ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.124:33343, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.126:43853, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.125:35747, [681ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.126:43853: Network error: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853, [683ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.124:33343: OK, [683ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.125:35747: Network error: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [685ms] delaying RPC due to: Service unavailable: Master config (127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [737ms] refreshing cache from master, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.124:33343, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.126:43853, [739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.125:35747, [741ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.126:43853: Network error: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853, [742ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.124:33343: OK, [744ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.125:35747: Network error: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [745ms] delaying RPC due to: Service unavailable: Master config (127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [837ms] refreshing cache from master, [837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.124:33343, [838ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.126:43853, [839ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.125:35747, [841ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.126:43853: Network error: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853, [842ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.124:33343: OK, [843ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.125:35747: Network error: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [845ms] delaying RPC due to: Service unavailable: Master config (127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [977ms] refreshing cache from master, [977ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.124:33343, [978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.126:43853, [979ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.125:35747, [983ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.126:43853: Network error: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853, [984ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.124:33343: OK, [985ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.125:35747: Network error: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [986ms] delaying RPC due to: Service unavailable: Master config (127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [1297ms] refreshing cache from master, [1297ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.124:33343, [1297ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.126:43853, [1299ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.125:35747, [1301ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.126:43853: Network error: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853, [1301ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.125:35747: Network error: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [1303ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.124:33343: OK, [1304ms] delaying RPC due to: Service unavailable: Master config (127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [1757ms] refreshing cache from master, [1757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.124:33343, [1758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.126:43853, [1758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.125:35747, [1761ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.126:43853: Network error: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853, [1761ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.125:35747: Network error: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [1762ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.124:33343: OK, [1763ms] delaying RPC due to: Service unavailable: Master config (127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853, [2696ms] refreshing cache from master, [2697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.124:33343, [2697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.126:43853, [2699ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.125:35747, [2700ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.126:43853: Network error: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853, [2701ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.124:33343: OK, [2702ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.125:35747: Network error: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [2702ms] delaying RPC due to: Service unavailable: Master config (127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [6717ms] refreshing cache from master, [6717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.124:33343, [6718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.126:43853, [6719ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.125:35747, [6720ms] too many traces: truncated at 100 traces, deferred=Deferred@1727502134(state=PENDING, result=null, callback=(continuation of Deferred@1998747194 after retry RPC after error@190042322) -> (continuation of Deferred@330704161 after retry RPC after error@1235377734) -> (continuation of Deferred@760574702 after retry RPC after error@940962518) -> (continuation of Deferred@447236056 after retry RPC after error@1092844197) -> (continuation of Deferred@1206362733 after retry RPC after error@1358134021) -> (continuation of Deferred@138240068 after retry RPC after error@1376896436) -> (continuation of Deferred@673177937 after retry RPC after error@1677696004) -> (continuation of Deferred@2359167 after retry RPC after error@1547937450) -> (continuation of Deferred@193057050 after retry RPC after error@344024018) -> (continuation of Deferred@189181498 after retry RPC after error@1248935694) -> (continuation of Deferred@1341992539 after retry RPC after error@1358777221) -> (continuation of Deferred@1853102302 after retry RPC after error@767480657) -> (continuation of Deferred@194490187 after retry RPC after error@822901983) -> (continuation of Deferred@2138058952 after retry RPC after error@613367852) -> (continuation of Deferred@307514881 after retry RPC after error@1080832528) -> (continuation of Deferred@1046789285 after retry RPC after error@1614021989) -> (continuation of Deferred@1344301860 after retry RPC after error@1912418178) -> (continuation of Deferred@1793537164 after retry RPC after error@708320152) -> (continuation of Deferred@2031234224 after retry RPC after error@1628151087) -> (continuation of Deferred@849057494 after retry RPC after error@1486204870) -> (continuation of Deferred@805990344 after retry RPC after error@223702531) -> (continuation of Deferred@1670804569 after retry RPC after error@369626226), errback=(continuation of Deferred@1998747194 after retry RPC after error@190042322) -> (continuation of Deferred@330704161 after retry RPC after error@1235377734) -> (continuation of Deferred@760574702 after retry RPC after error@940962518) -> (continuation of Deferred@447236056 after retry RPC after error@1092844197) -> (continuation of Deferred@1206362733 after retry RPC after error@1358134021) -> (continuation of Deferred@138240068 after retry RPC after error@1376896436) -> (continuation of Deferred@673177937 after retry RPC after error@1677696004) -> (continuation of Deferred@2359167 after retry RPC after error@1547937450) -> (continuation of Deferred@193057050 after retry RPC after error@344024018) -> (continuation of Deferred@189181498 after retry RPC after error@1248935694) -> (continuation of Deferred@1341992539 after retry RPC after error@1358777221) -> (continuation of Deferred@1853102302 after retry RPC after error@767480657) -> (continuation of Deferred@194490187 after retry RPC after error@822901983) -> (continuation of Deferred@2138058952 after retry RPC after error@613367852) -> (continuation of Deferred@307514881 after retry RPC after error@1080832528) -> (continuation of Deferred@1046789285 after retry RPC after error@1614021989) -> (continuation of Deferred@1344301860 after retry RPC after error@1912418178) -> (continuation of Deferred@1793537164 after retry RPC after error@708320152) -> (continuation of Deferred@2031234224 after retry RPC after error@1628151087) -> (continuation of Deferred@849057494 after retry RPC after error@1486204870) -> (continuation of Deferred@805990344 after retry RPC after error@223702531) -> (continuation of Deferred@1670804569 after retry RPC after error@369626226)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.access$200(DefaultPromise.java:35) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise$1.run(DefaultPromise.java:503) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.4.48.125:35747
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=23, TimeoutTracker(timeout=30000, elapsed=27546), Traces: [0ms] refreshing cache from master, [51ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.124:33343, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.126:43853, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.125:35747, [192ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.126:43853: Network error: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853, [256ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.125:35747: Network error: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [531ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.124:33343: OK, [536ms] delaying RPC due to: Service unavailable: Master config (127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [557ms] refreshing cache from master, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.124:33343, [560ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.126:43853, [562ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.125:35747, [563ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.124:33343: OK, [564ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.126:43853: Network error: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853, [567ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.125:35747: Network error: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [569ms] delaying RPC due to: Service unavailable: Master config (127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [577ms] refreshing cache from master, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.124:33343, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.126:43853, [579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.125:35747, [581ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.126:43853: Network error: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853, [583ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.124:33343: OK, [583ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.125:35747: Network error: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [584ms] delaying RPC due to: Service unavailable: Master config (127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [596ms] refreshing cache from master, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.124:33343, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.126:43853, [600ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.125:35747, [600ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.126:43853: Network error: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853, [601ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.124:33343: OK, [604ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.125:35747: Network error: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [605ms] delaying RPC due to: Service unavailable: Master config (127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [636ms] refreshing cache from master, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.124:33343, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.126:43853, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.125:35747, [641ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.126:43853: Network error: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853, [642ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.124:33343: OK, [643ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.125:35747: Network error: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [644ms] delaying RPC due to: Service unavailable: Master config (127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [677ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.124:33343, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.126:43853, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.125:35747, [681ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.126:43853: Network error: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853, [683ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.124:33343: OK, [683ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.125:35747: Network error: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [685ms] delaying RPC due to: Service unavailable: Master config (127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [737ms] refreshing cache from master, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.124:33343, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.126:43853, [739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.125:35747, [741ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.126:43853: Network error: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853, [742ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.124:33343: OK, [744ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.125:35747: Network error: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [745ms] delaying RPC due to: Service unavailable: Master config (127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [837ms] refreshing cache from master, [837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.124:33343, [838ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.126:43853, [839ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.125:35747, [841ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.126:43853: Network error: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853, [842ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.124:33343: OK, [843ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.125:35747: Network error: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [845ms] delaying RPC due to: Service unavailable: Master config (127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [977ms] refreshing cache from master, [977ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.124:33343, [978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.126:43853, [979ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.125:35747, [983ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.126:43853: Network error: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853, [984ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.124:33343: OK, [985ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.125:35747: Network error: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [986ms] delaying RPC due to: Service unavailable: Master config (127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [1297ms] refreshing cache from master, [1297ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.124:33343, [1297ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.126:43853, [1299ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.125:35747, [1301ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.126:43853: Network error: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853, [1301ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.125:35747: Network error: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [1303ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.124:33343: OK, [1304ms] delaying RPC due to: Service unavailable: Master config (127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [1757ms] refreshing cache from master, [1757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.124:33343, [1758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.126:43853, [1758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.125:35747, [1761ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.126:43853: Network error: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853, [1761ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.125:35747: Network error: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [1762ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.124:33343: OK, [1763ms] delaying RPC due to: Service unavailable: Master config (127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853, [2696ms] refreshing cache from master, [2697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.124:33343, [2697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.126:43853, [2699ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.125:35747, [2700ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.126:43853: Network error: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853, [2701ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.124:33343: OK, [2702ms] Sub RPC ConnectToMaster: received response from server master-127.4.48.125:35747: Network error: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [2702ms] delaying RPC due to: Service unavailable: Master config (127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747, [6717ms] refreshing cache from master, [6717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.124:33343, [6718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.126:43853, [6719ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.48.125:35747, [6720ms] too many traces: truncated at 100 traces, deferred=Deferred@1727502134(state=PENDING, result=null, callback=(continuation of Deferred@1998747194 after retry RPC after error@190042322) -> (continuation of Deferred@330704161 after retry RPC after error@1235377734) -> (continuation of Deferred@760574702 after retry RPC after error@940962518) -> (continuation of Deferred@447236056 after retry RPC after error@1092844197) -> (continuation of Deferred@1206362733 after retry RPC after error@1358134021) -> (continuation of Deferred@138240068 after retry RPC after error@1376896436) -> (continuation of Deferred@673177937 after retry RPC after error@1677696004) -> (continuation of Deferred@2359167 after retry RPC after error@1547937450) -> (continuation of Deferred@193057050 after retry RPC after error@344024018) -> (continuation of Deferred@189181498 after retry RPC after error@1248935694) -> (continuation of Deferred@1341992539 after retry RPC after error@1358777221) -> (continuation of Deferred@1853102302 after retry RPC after error@767480657) -> (continuation of Deferred@194490187 after retry RPC after error@822901983) -> (continuation of Deferred@2138058952 after retry RPC after error@613367852) -> (continuation of Deferred@307514881 after retry RPC after error@1080832528) -> (continuation of Deferred@1046789285 after retry RPC after error@1614021989) -> (continuation of Deferred@1344301860 after retry RPC after error@1912418178) -> (continuation of Deferred@1793537164 after retry RPC after error@708320152) -> (continuation of Deferred@2031234224 after retry RPC after error@1628151087) -> (continuation of Deferred@849057494 after retry RPC after error@1486204870) -> (continuation of Deferred@805990344 after retry RPC after error@223702531) -> (continuation of Deferred@1670804569 after retry RPC after error@369626226), errback=(continuation of Deferred@1998747194 after retry RPC after error@190042322) -> (continuation of Deferred@330704161 after retry RPC after error@1235377734) -> (continuation of Deferred@760574702 after retry RPC after error@940962518) -> (continuation of Deferred@447236056 after retry RPC after error@1092844197) -> (continuation of Deferred@1206362733 after retry RPC after error@1358134021) -> (continuation of Deferred@138240068 after retry RPC after error@1376896436) -> (continuation of Deferred@673177937 after retry RPC after error@1677696004) -> (continuation of Deferred@2359167 after retry RPC after error@1547937450) -> (continuation of Deferred@193057050 after retry RPC after error@344024018) -> (continuation of Deferred@189181498 after retry RPC after error@1248935694) -> (continuation of Deferred@1341992539 after retry RPC after error@1358777221) -> (continuation of Deferred@1853102302 after retry RPC after error@767480657) -> (continuation of Deferred@194490187 after retry RPC after error@822901983) -> (continuation of Deferred@2138058952 after retry RPC after error@613367852) -> (continuation of Deferred@307514881 after retry RPC after error@1080832528) -> (continuation of Deferred@1046789285 after retry RPC after error@1614021989) -> (continuation of Deferred@1344301860 after retry RPC after error@1912418178) -> (continuation of Deferred@1793537164 after retry RPC after error@708320152) -> (continuation of Deferred@2031234224 after retry RPC after error@1628151087) -> (continuation of Deferred@849057494 after retry RPC after error@1486204870) -> (continuation of Deferred@805990344 after retry RPC after error@223702531) -> (continuation of Deferred@1670804569 after retry RPC after error@369626226)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.4.48.124:33343,127.4.48.126:43853,127.4.48.125:35747) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.126:43853(127.4.48.126:43853): Connection refused: /127.4.48.126:43853,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.48.125:35747(127.4.48.125:35747): Connection refused: /127.4.48.125:35747
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557)
at io.netty.util.concurrent.DefaultPromise.access$200(DefaultPromise.java:35)
at io.netty.util.concurrent.DefaultPromise$1.run(DefaultPromise.java:503)
at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174)
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167)
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569)
... 5 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.4.48.125:35747
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)