11:44:50.125 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
11:44:50.136 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskst9njx/test-tmp
11:44:50.138 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
11:44:50.142 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskst9njx/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
11:44:50.345 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
11:44:50.503 [DEBUG - main] (MiniKuduCluster.java:195) Response:
11:44:50.504 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
11:44:50.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:50.508896 10373 mini_oidc.cc:152] Starting JWKS server
11:44:50.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:50.512640 10373 webserver.cc:460] Webserver started at http://127.0.0.1:37517/ using document root <none> and password file <none>
11:44:50.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:50.513043 10373 mini_oidc.cc:189] Starting OIDC Discovery server
11:44:50.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:50.514771 10373 webserver.cc:460] Webserver started at http://127.0.0.1:40953/ using document root <none> and password file <none>
11:44:50.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T11:44:50Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
11:44:50.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T11:44:50Z Disabled control of system clock
11:44:50.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:50.560834 10373 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskst9njx/build/tsan/bin/kudu
11:44:50.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskst9njx/build/tsan/bin/kudu
11:44:50.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/master-0/wal
11:44:50.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/master-0/data
11:44:50.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:44:50.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:44:50.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:44:50.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:44:50.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:44:50.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:44:50.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/master-0/logs
11:44:50.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/master-0/data/info.pb
11:44:50.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:44:50.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:44:50.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:44:50.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:44:50.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:44:50.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:44:50.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:44:50.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:44:50.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
11:44:50.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:44:50.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:44:50.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:44:50.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.33.126:37191
11:44:50.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.33.126
11:44:50.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:44:50.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.33.84:35767
11:44:50.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:44:50.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:44:50.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:44:50.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:44:50.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.33.126:37191,127.10.33.125:46227,127.10.33.124:36931
11:44:50.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:44:50.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37517/jwks.json with env {}
11:44:50.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:50.859423 10385 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:44:50.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:50.859959 10385 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:44:50.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:50.860623 10385 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:44:50.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:50.891394 10385 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
11:44:50.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:50.891726 10385 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:44:50.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:50.891925 10385 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
11:44:50.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:50.892206 10385 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
11:44:50.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:50.892503 10385 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
11:44:50.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:50.892729 10385 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37517/jwks.json
11:44:50.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:50.926388 10385 master_runner.cc:386] Master server non-default flags:
11:44:50.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:44:50.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.33.84:35767
11:44:50.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:44:50.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:44:50.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/master-0/data
11:44:50.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/master-0/wal
11:44:50.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:44:50.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.33.126:37191,127.10.33.125:46227,127.10.33.124:36931
11:44:50.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:44:50.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:44:50.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:44:50.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.33.126:37191
11:44:50.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:44:50.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:44:50.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:44:50.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37517/jwks.json
11:44:50.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:44:50.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:44:50.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/master-0/data/info.pb
11:44:50.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.33.126
11:44:50.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:44:50.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:44:50.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:44:50.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:44:50.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:44:50.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:44:50.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/master-0/logs
11:44:50.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:44:50.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:44:50.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:44:50.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
11:44:50.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:44:50.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:44:50.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:44:50.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 10:52:50 UTC on ccycloud.quasar-peqjre.root.comops.site
11:44:50.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679900
11:44:50.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:44:50.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:50.927747 10385 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:44:50.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:50.929819 10385 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:44:50.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:50.947824 10391 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:44:52.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:50.947816 10392 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:44:52.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:50.949239 10394 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:44:52.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:52.114288 10393 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
11:44:52.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:52.114506 10385 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
11:44:52.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:52.118364 10385 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:44:52.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:52.121371 10385 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:44:52.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:52.122836 10385 hybrid_clock.cc:648] HybridClock initialized: now 1790682292122789 us; error 41 us; skew 500 ppm
11:44:52.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:52.123719 10385 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:44:52.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:52.130453 10385 webserver.cc:460] Webserver started at http://127.10.33.126:35907/ using document root <none> and password file <none>
11:44:52.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:52.131246 10385 fs_manager.cc:359] Metadata directory not provided
11:44:52.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:52.131402 10385 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:44:52.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:52.131811 10385 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
11:44:52.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:52.136713 10385 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/master-0/data/instance:
11:44:52.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fe53874142bb48fc95f6360091185405"
11:44:52.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:44:52 on dist-test-slave-8d15"
11:44:52.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:52.137789 10385 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/master-0/wal/instance:
11:44:52.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fe53874142bb48fc95f6360091185405"
11:44:52.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:44:52 on dist-test-slave-8d15"
11:44:52.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:52.144927 10385 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.004s sys 0.004s
11:44:52.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:52.150564 10401 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:44:52.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:52.151758 10385 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
11:44:52.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:52.152132 10385 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/master-0/data,/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/master-0/wal
11:44:52.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fe53874142bb48fc95f6360091185405"
11:44:52.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:44:52 on dist-test-slave-8d15"
11:44:52.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:52.152520 10385 fs_report.cc:383] FS layout report
11:44:52.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:44:52.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/master-0/wal
11:44:52.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/master-0/wal
11:44:52.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/master-0/data/data
11:44:52.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:44:52.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:44:52.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:44:52.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:44:52.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:44:52.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:44:52.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:44:52.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:44:52.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:44:52.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:44:52.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:44:52.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:52.187829 10385 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:44:52.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:52.189571 10385 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:44:52.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:52.190112 10385 kserver.cc:163] Server-wide thread pool size limit: 3276
11:44:52.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:52.257598 10385 rpc_server.cc:307] RPC server started. Bound to: 127.10.33.126:37191
11:44:52.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:52.257695 10452 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.33.126:37191 every 8 connection(s)
11:44:52.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:52.260443 10385 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/master-0/data/info.pb
11:44:52.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:52.265884 10453 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
11:44:52.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:52.268355 10373 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskst9njx/build/tsan/bin/kudu as pid 10385
11:44:52.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:52.268828 10373 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/master-0/wal/instance
11:44:52.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:52.272305 10373 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskst9njx/build/tsan/bin/kudu
11:44:52.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskst9njx/build/tsan/bin/kudu
11:44:52.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/master-1/wal
11:44:52.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/master-1/data
11:44:52.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:44:52.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:44:52.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:44:52.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:44:52.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:44:52.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:44:52.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/master-1/logs
11:44:52.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/master-1/data/info.pb
11:44:52.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:44:52.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:44:52.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:44:52.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:44:52.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:44:52.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:44:52.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:44:52.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:44:52.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
11:44:52.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:44:52.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:44:52.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:44:52.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.33.125:46227
11:44:52.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.33.125
11:44:52.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:44:52.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.33.84:35767
11:44:52.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:44:52.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:44:52.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:44:52.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:44:52.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.33.126:37191,127.10.33.125:46227,127.10.33.124:36931
11:44:52.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:44:52.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37517/jwks.json with env {}
11:44:52.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:52.280332 10453 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.33.126" port: 37191 } has no permanent_uuid. Determining permanent_uuid...
11:44:52.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:52.311643 10453 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.33.125" port: 46227 } has no permanent_uuid. Determining permanent_uuid...
11:44:52.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:52.314035 10403 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.33.125:46227: connect: Connection refused (error 111)
11:44:52.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:52.317946 10453 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.33.125:46227: Network error: Client connection negotiation failed: client connection to 127.10.33.125:46227: connect: Connection refused (error 111)
11:44:52.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:52.367422 10453 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.33.125" port: 46227 } attempt: 1
11:44:52.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:52.371636 10453 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.33.125:46227: Network error: Client connection negotiation failed: client connection to 127.10.33.125:46227: connect: Connection refused (error 111)
11:44:52.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:52.440109 10453 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.33.125" port: 46227 } attempt: 2
11:44:52.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:52.444455 10453 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.33.125:46227: Network error: Client connection negotiation failed: client connection to 127.10.33.125:46227: connect: Connection refused (error 111)
11:44:52.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:52.535984 10453 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.33.125" port: 46227 } attempt: 3
11:44:52.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:52.539760 10453 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.33.125:46227: Network error: Client connection negotiation failed: client connection to 127.10.33.125:46227: connect: Connection refused (error 111)
11:44:52.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:52.583381 10455 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:44:52.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:52.583998 10455 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:44:52.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:52.584735 10455 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:44:52.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:52.613900 10455 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
11:44:52.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:52.614166 10455 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:44:52.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:52.614346 10455 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
11:44:52.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:52.614537 10455 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
11:44:52.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:52.614782 10455 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
11:44:52.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:52.615010 10455 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37517/jwks.json
11:44:52.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:52.653275 10455 master_runner.cc:386] Master server non-default flags:
11:44:52.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:44:52.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.33.84:35767
11:44:52.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:44:52.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:44:52.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/master-1/data
11:44:52.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/master-1/wal
11:44:52.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:44:52.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.33.126:37191,127.10.33.125:46227,127.10.33.124:36931
11:44:52.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:44:52.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:44:52.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:44:52.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.33.125:46227
11:44:52.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:44:52.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:44:52.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:44:52.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37517/jwks.json
11:44:52.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:44:52.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:44:52.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/master-1/data/info.pb
11:44:52.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.33.125
11:44:52.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:44:52.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:44:52.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:44:52.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:44:52.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:44:52.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:44:52.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/master-1/logs
11:44:52.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:44:52.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:44:52.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:44:52.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
11:44:52.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:44:52.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:44:52.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:44:52.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 10:52:50 UTC on ccycloud.quasar-peqjre.root.comops.site
11:44:52.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679900
11:44:52.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:44:52.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:52.654528 10455 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:44:52.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:52.656704 10455 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:44:52.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:52.675864 10464 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:44:52.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:52.683139 10453 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.33.125" port: 46227 } attempt: 4
11:44:52.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:52.687198 10453 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.33.125:46227: Network error: Client connection negotiation failed: client connection to 127.10.33.125:46227: connect: Connection refused (error 111)
11:44:52.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:52.986639 10453 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.33.125" port: 46227 } attempt: 5
11:44:52.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:52.990696 10453 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.33.125:46227: Network error: Client connection negotiation failed: client connection to 127.10.33.125:46227: connect: Connection refused (error 111)
11:44:53.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:53.538285 10453 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.33.125" port: 46227 } attempt: 6
11:44:53.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:53.545378 10453 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.33.125:46227: Network error: Client connection negotiation failed: client connection to 127.10.33.125:46227: connect: Connection refused (error 111)
11:44:53.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:52.675875 10465 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:44:53.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:52.677017 10467 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:44:53.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:52.677580 10455 server_base.cc:1034] running on GCE node
11:44:53.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:53.861343 10455 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:44:53.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:53.864040 10455 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:44:53.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:53.865366 10455 hybrid_clock.cc:648] HybridClock initialized: now 1790682293865320 us; error 67 us; skew 500 ppm
11:44:53.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:53.866153 10455 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:44:53.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:53.872493 10455 webserver.cc:460] Webserver started at http://127.10.33.125:33541/ using document root <none> and password file <none>
11:44:53.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:53.873477 10455 fs_manager.cc:359] Metadata directory not provided
11:44:53.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:53.873654 10455 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:44:53.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:53.874015 10455 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
11:44:53.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:53.878602 10455 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/master-1/data/instance:
11:44:53.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "23670df4991843518252f60c82410195"
11:44:53.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:44:53 on dist-test-slave-8d15"
11:44:53.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:53.879760 10455 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/master-1/wal/instance:
11:44:53.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "23670df4991843518252f60c82410195"
11:44:53.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:44:53 on dist-test-slave-8d15"
11:44:53.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:53.886739 10455 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.005s sys 0.000s
11:44:53.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:53.892603 10476 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:44:53.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:53.893615 10455 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
11:44:53.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:53.894018 10455 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/master-1/data,/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/master-1/wal
11:44:53.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "23670df4991843518252f60c82410195"
11:44:53.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:44:53 on dist-test-slave-8d15"
11:44:53.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:53.894366 10455 fs_report.cc:383] FS layout report
11:44:53.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:44:53.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/master-1/wal
11:44:53.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/master-1/wal
11:44:53.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/master-1/data/data
11:44:53.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:44:53.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:44:53.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:44:53.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:44:53.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:44:53.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:44:53.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:44:53.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:44:53.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:44:53.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:44:53.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:44:53.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:53.932763 10455 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:44:53.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:53.934240 10455 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:44:53.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:53.934751 10455 kserver.cc:163] Server-wide thread pool size limit: 3276
11:44:54.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:54.004859 10455 rpc_server.cc:307] RPC server started. Bound to: 127.10.33.125:46227
11:44:54.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:54.004938 10527 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.33.125:46227 every 8 connection(s)
11:44:54.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:54.007777 10455 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/master-1/data/info.pb
11:44:54.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:54.011572 10373 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskst9njx/build/tsan/bin/kudu as pid 10455
11:44:54.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:54.012027 10373 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/master-1/wal/instance
11:44:54.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:54.013613 10528 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
11:44:54.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:54.014940 10373 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskst9njx/build/tsan/bin/kudu
11:44:54.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskst9njx/build/tsan/bin/kudu
11:44:54.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/master-2/wal
11:44:54.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/master-2/data
11:44:54.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:44:54.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:44:54.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:44:54.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:44:54.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:44:54.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:44:54.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/master-2/logs
11:44:54.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/master-2/data/info.pb
11:44:54.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:44:54.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:44:54.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:44:54.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:44:54.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:44:54.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:44:54.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:44:54.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:44:54.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
11:44:54.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:44:54.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:44:54.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:44:54.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.33.124:36931
11:44:54.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.33.124
11:44:54.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:44:54.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.33.84:35767
11:44:54.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:44:54.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:44:54.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:44:54.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:44:54.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.33.126:37191,127.10.33.125:46227,127.10.33.124:36931
11:44:54.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:44:54.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37517/jwks.json with env {}
11:44:54.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:54.029150 10528 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.33.126" port: 37191 } has no permanent_uuid. Determining permanent_uuid...
11:44:54.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:54.056311 10528 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.33.125" port: 46227 } has no permanent_uuid. Determining permanent_uuid...
11:44:54.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:54.071326 10528 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.33.124" port: 36931 } has no permanent_uuid. Determining permanent_uuid...
11:44:54.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:54.074342 10477 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.33.124:36931: connect: Connection refused (error 111)
11:44:54.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:54.078415 10528 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.33.124:36931: Network error: Client connection negotiation failed: client connection to 127.10.33.124:36931: connect: Connection refused (error 111)
11:44:54.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:54.127959 10528 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.33.124" port: 36931 } attempt: 1
11:44:54.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:54.132241 10528 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.33.124:36931: Network error: Client connection negotiation failed: client connection to 127.10.33.124:36931: connect: Connection refused (error 111)
11:44:54.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:54.200672 10528 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.33.124" port: 36931 } attempt: 2
11:44:54.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:54.204421 10528 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.33.124:36931: Network error: Client connection negotiation failed: client connection to 127.10.33.124:36931: connect: Connection refused (error 111)
11:44:54.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:54.295887 10528 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.33.124" port: 36931 } attempt: 3
11:44:54.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:54.299623 10528 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.33.124:36931: Network error: Client connection negotiation failed: client connection to 127.10.33.124:36931: connect: Connection refused (error 111)
11:44:54.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:54.330355 10530 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:44:54.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:54.330847 10530 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:44:54.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:54.331339 10530 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:44:54.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:54.361064 10530 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
11:44:54.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:54.361357 10530 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:44:54.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:54.361635 10530 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
11:44:54.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:54.361867 10530 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
11:44:54.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:54.362072 10530 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
11:44:54.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:54.362214 10530 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37517/jwks.json
11:44:54.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:54.395222 10530 master_runner.cc:386] Master server non-default flags:
11:44:54.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:44:54.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.33.84:35767
11:44:54.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:44:54.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:44:54.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/master-2/data
11:44:54.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/master-2/wal
11:44:54.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:44:54.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.33.126:37191,127.10.33.125:46227,127.10.33.124:36931
11:44:54.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:44:54.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:44:54.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:44:54.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.33.124:36931
11:44:54.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:44:54.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:44:54.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:44:54.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37517/jwks.json
11:44:54.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:44:54.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:44:54.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/master-2/data/info.pb
11:44:54.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.33.124
11:44:54.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:44:54.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:44:54.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:44:54.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:44:54.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:44:54.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:44:54.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/master-2/logs
11:44:54.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:44:54.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:44:54.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:44:54.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
11:44:54.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:44:54.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:44:54.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:44:54.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 10:52:50 UTC on ccycloud.quasar-peqjre.root.comops.site
11:44:54.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679900
11:44:54.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:44:54.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:54.396397 10530 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:44:54.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:54.398411 10530 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:44:54.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:54.417565 10540 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:44:54.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:54.443040 10528 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.33.124" port: 36931 } attempt: 4
11:44:54.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:54.446458 10528 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.33.124:36931: Network error: Client connection negotiation failed: client connection to 127.10.33.124:36931: connect: Connection refused (error 111)
11:44:54.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:54.605955 10453 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.33.125" port: 46227 } attempt: 7
11:44:54.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:54.619954 10453 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.33.124" port: 36931 } has no permanent_uuid. Determining permanent_uuid...
11:44:54.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:54.625586 10453 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.33.124:36931: Network error: Client connection negotiation failed: client connection to 127.10.33.124:36931: connect: Connection refused (error 111)
11:44:54.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:54.684041 10453 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.33.124" port: 36931 } attempt: 1
11:44:54.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:54.688742 10453 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.33.124:36931: Network error: Client connection negotiation failed: client connection to 127.10.33.124:36931: connect: Connection refused (error 111)
11:44:54.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:54.746107 10528 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.33.124" port: 36931 } attempt: 5
11:44:54.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:54.751322 10528 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.33.124:36931: Network error: Client connection negotiation failed: client connection to 127.10.33.124:36931: connect: Connection refused (error 111)
11:44:54.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:54.770159 10453 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.33.124" port: 36931 } attempt: 2
11:44:54.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:54.774356 10453 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.33.124:36931: Network error: Client connection negotiation failed: client connection to 127.10.33.124:36931: connect: Connection refused (error 111)
11:44:54.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:54.859728 10453 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.33.124" port: 36931 } attempt: 3
11:44:54.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:54.863891 10453 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.33.124:36931: Network error: Client connection negotiation failed: client connection to 127.10.33.124:36931: connect: Connection refused (error 111)
11:44:55.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:55.004298 10453 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.33.124" port: 36931 } attempt: 4
11:44:55.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:55.009477 10453 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.33.124:36931: Network error: Client connection negotiation failed: client connection to 127.10.33.124:36931: connect: Connection refused (error 111)
11:44:55.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:55.292966 10453 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.33.124" port: 36931 } attempt: 5
11:44:55.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:55.297513 10453 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.33.124:36931: Network error: Client connection negotiation failed: client connection to 127.10.33.124:36931: connect: Connection refused (error 111)
11:44:55.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:55.298841 10528 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.33.124" port: 36931 } attempt: 6
11:44:55.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:55.304949 10528 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.33.124:36931: Network error: Client connection negotiation failed: client connection to 127.10.33.124:36931: connect: Connection refused (error 111)
11:44:55.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:54.420598 10541 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:44:55.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:54.419966 10543 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:44:55.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:54.423681 10530 server_base.cc:1034] running on GCE node
11:44:55.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:55.618964 10530 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:44:55.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:55.622066 10530 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:44:55.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:55.623466 10530 hybrid_clock.cc:648] HybridClock initialized: now 1790682295623443 us; error 44 us; skew 500 ppm
11:44:55.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:55.624317 10530 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:44:55.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:55.630858 10530 webserver.cc:460] Webserver started at http://127.10.33.124:42815/ using document root <none> and password file <none>
11:44:55.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:55.631731 10530 fs_manager.cc:359] Metadata directory not provided
11:44:55.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:55.631902 10530 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:44:55.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:55.632339 10530 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
11:44:55.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:55.637199 10530 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/master-2/data/instance:
11:44:55.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d2fe976495bf4db09c551bceed68ed96"
11:44:55.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:44:55 on dist-test-slave-8d15"
11:44:55.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:55.638195 10530 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/master-2/wal/instance:
11:44:55.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d2fe976495bf4db09c551bceed68ed96"
11:44:55.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:44:55 on dist-test-slave-8d15"
11:44:55.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:55.645466 10530 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.001s sys 0.008s
11:44:55.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:55.650731 10555 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:44:55.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:55.651722 10530 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
11:44:55.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:55.652074 10530 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/master-2/data,/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/master-2/wal
11:44:55.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d2fe976495bf4db09c551bceed68ed96"
11:44:55.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:44:55 on dist-test-slave-8d15"
11:44:55.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:55.652403 10530 fs_report.cc:383] FS layout report
11:44:55.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:44:55.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/master-2/wal
11:44:55.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/master-2/wal
11:44:55.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/master-2/data/data
11:44:55.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:44:55.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:44:55.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:44:55.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:44:55.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:44:55.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:44:55.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:44:55.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:44:55.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:44:55.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:44:55.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:44:55.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:55.703384 10530 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:44:55.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:55.704909 10530 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:44:55.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:55.705415 10530 kserver.cc:163] Server-wide thread pool size limit: 3276
11:44:55.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:55.772228 10530 rpc_server.cc:307] RPC server started. Bound to: 127.10.33.124:36931
11:44:55.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:55.772295 10606 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.33.124:36931 every 8 connection(s)
11:44:55.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:55.774926 10530 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/master-2/data/info.pb
11:44:55.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:55.780769 10607 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
11:44:55.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:55.785332 10373 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskst9njx/build/tsan/bin/kudu as pid 10530
11:44:55.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:55.785637 10373 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/master-2/wal/instance
11:44:55.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:55.792722 10607 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.33.126" port: 37191 } has no permanent_uuid. Determining permanent_uuid...
11:44:55.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:55.827672 10607 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.33.125" port: 46227 } has no permanent_uuid. Determining permanent_uuid...
11:44:55.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:55.840117 10607 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.33.124" port: 36931 } has no permanent_uuid. Determining permanent_uuid...
11:44:55.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:55.849874 10453 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.33.124" port: 36931 } attempt: 6
11:44:55.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:55.861066 10607 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d2fe976495bf4db09c551bceed68ed96: Bootstrap starting.
11:44:55.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:55.868005 10607 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P d2fe976495bf4db09c551bceed68ed96: Neither blocks nor log segments found. Creating new log.
11:44:55.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:55.869375 10453 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P fe53874142bb48fc95f6360091185405: Bootstrap starting.
11:44:55.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:55.870117 10607 log.cc:824] T 00000000000000000000000000000000 P d2fe976495bf4db09c551bceed68ed96: Log is configured to *not* fsync() on all Append() calls
11:44:55.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:55.876017 10453 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P fe53874142bb48fc95f6360091185405: Neither blocks nor log segments found. Creating new log.
11:44:55.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:55.876556 10607 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d2fe976495bf4db09c551bceed68ed96: No bootstrap required, opened a new log
11:44:55.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:55.877840 10453 log.cc:824] T 00000000000000000000000000000000 P fe53874142bb48fc95f6360091185405: Log is configured to *not* fsync() on all Append() calls
11:44:55.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:55.882870 10453 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P fe53874142bb48fc95f6360091185405: No bootstrap required, opened a new log
11:44:55.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:55.897904 10607 raft_consensus.cc:348] T 00000000000000000000000000000000 P d2fe976495bf4db09c551bceed68ed96 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fe53874142bb48fc95f6360091185405" member_type: VOTER last_known_addr { host: "127.10.33.126" port: 37191 } } peers { permanent_uuid: "23670df4991843518252f60c82410195" member_type: VOTER last_known_addr { host: "127.10.33.125" port: 46227 } } peers { permanent_uuid: "d2fe976495bf4db09c551bceed68ed96" member_type: VOTER last_known_addr { host: "127.10.33.124" port: 36931 } }
11:44:55.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:55.898463 10607 raft_consensus.cc:374] T 00000000000000000000000000000000 P d2fe976495bf4db09c551bceed68ed96 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
11:44:55.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:55.898655 10607 raft_consensus.cc:729] T 00000000000000000000000000000000 P d2fe976495bf4db09c551bceed68ed96 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d2fe976495bf4db09c551bceed68ed96, State: Initialized, Role: FOLLOWER
11:44:55.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:55.899432 10607 consensus_queue.cc:260] T 00000000000000000000000000000000 P d2fe976495bf4db09c551bceed68ed96 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fe53874142bb48fc95f6360091185405" member_type: VOTER last_known_addr { host: "127.10.33.126" port: 37191 } } peers { permanent_uuid: "23670df4991843518252f60c82410195" member_type: VOTER last_known_addr { host: "127.10.33.125" port: 46227 } } peers { permanent_uuid: "d2fe976495bf4db09c551bceed68ed96" member_type: VOTER last_known_addr { host: "127.10.33.124" port: 36931 } }
11:44:55.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:55.902562 10620 sys_catalog.cc:455] T 00000000000000000000000000000000 P d2fe976495bf4db09c551bceed68ed96 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fe53874142bb48fc95f6360091185405" member_type: VOTER last_known_addr { host: "127.10.33.126" port: 37191 } } peers { permanent_uuid: "23670df4991843518252f60c82410195" member_type: VOTER last_known_addr { host: "127.10.33.125" port: 46227 } } peers { permanent_uuid: "d2fe976495bf4db09c551bceed68ed96" member_type: VOTER last_known_addr { host: "127.10.33.124" port: 36931 } } }
11:44:55.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:55.903750 10620 sys_catalog.cc:458] T 00000000000000000000000000000000 P d2fe976495bf4db09c551bceed68ed96 [sys.catalog]: This master's current role is: FOLLOWER
11:44:55.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:55.903765 10453 raft_consensus.cc:348] T 00000000000000000000000000000000 P fe53874142bb48fc95f6360091185405 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fe53874142bb48fc95f6360091185405" member_type: VOTER last_known_addr { host: "127.10.33.126" port: 37191 } } peers { permanent_uuid: "23670df4991843518252f60c82410195" member_type: VOTER last_known_addr { host: "127.10.33.125" port: 46227 } } peers { permanent_uuid: "d2fe976495bf4db09c551bceed68ed96" member_type: VOTER last_known_addr { host: "127.10.33.124" port: 36931 } }
11:44:55.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:55.904451 10607 sys_catalog.cc:564] T 00000000000000000000000000000000 P d2fe976495bf4db09c551bceed68ed96 [sys.catalog]: configured and running, proceeding with master startup.
11:44:55.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:55.905174 10453 raft_consensus.cc:374] T 00000000000000000000000000000000 P fe53874142bb48fc95f6360091185405 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
11:44:55.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:55.905639 10453 raft_consensus.cc:729] T 00000000000000000000000000000000 P fe53874142bb48fc95f6360091185405 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fe53874142bb48fc95f6360091185405, State: Initialized, Role: FOLLOWER
11:44:55.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:55.906454 10453 consensus_queue.cc:260] T 00000000000000000000000000000000 P fe53874142bb48fc95f6360091185405 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fe53874142bb48fc95f6360091185405" member_type: VOTER last_known_addr { host: "127.10.33.126" port: 37191 } } peers { permanent_uuid: "23670df4991843518252f60c82410195" member_type: VOTER last_known_addr { host: "127.10.33.125" port: 46227 } } peers { permanent_uuid: "d2fe976495bf4db09c551bceed68ed96" member_type: VOTER last_known_addr { host: "127.10.33.124" port: 36931 } }
11:44:55.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:55.909716 10622 sys_catalog.cc:455] T 00000000000000000000000000000000 P fe53874142bb48fc95f6360091185405 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fe53874142bb48fc95f6360091185405" member_type: VOTER last_known_addr { host: "127.10.33.126" port: 37191 } } peers { permanent_uuid: "23670df4991843518252f60c82410195" member_type: VOTER last_known_addr { host: "127.10.33.125" port: 46227 } } peers { permanent_uuid: "d2fe976495bf4db09c551bceed68ed96" member_type: VOTER last_known_addr { host: "127.10.33.124" port: 36931 } } }
11:44:55.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:55.910714 10622 sys_catalog.cc:458] T 00000000000000000000000000000000 P fe53874142bb48fc95f6360091185405 [sys.catalog]: This master's current role is: FOLLOWER
11:44:55.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:55.911486 10453 sys_catalog.cc:564] T 00000000000000000000000000000000 P fe53874142bb48fc95f6360091185405 [sys.catalog]: configured and running, proceeding with master startup.
11:44:55.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:55.939746 10637 catalog_manager.cc:1558] T 00000000000000000000000000000000 P d2fe976495bf4db09c551bceed68ed96: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:44:55.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:55.940383 10637 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:44:55.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:55.949928 10407 tablet.cc:2369] T 00000000000000000000000000000000 P fe53874142bb48fc95f6360091185405: Can't schedule compaction. Clean time has not been advanced past its initial value.
11:44:55.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:55.960132 10561 tablet.cc:2369] T 00000000000000000000000000000000 P d2fe976495bf4db09c551bceed68ed96: Can't schedule compaction. Clean time has not been advanced past its initial value.
11:44:55.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:55.929529 10530 master_runner.cc:186] Error getting master registration for 127.10.33.126:37191: OK, instance_id { permanent_uuid: "fe53874142bb48fc95f6360091185405" instance_seqno: 1790682292239250 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
11:44:55.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:55.965487 10530 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
11:44:55.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:55.980891 10644 catalog_manager.cc:1558] T 00000000000000000000000000000000 P fe53874142bb48fc95f6360091185405: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:44:55.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:55.981550 10644 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:44:55.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:55.957141 10385 master_runner.cc:186] Error getting master registration for 127.10.33.125:46227: OK, instance_id { permanent_uuid: "23670df4991843518252f60c82410195" instance_seqno: 1790682293986040 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
11:44:55.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:55.989202 10385 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
11:44:56.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.051222 10620 raft_consensus.cc:482] T 00000000000000000000000000000000 P d2fe976495bf4db09c551bceed68ed96 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
11:44:56.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.051822 10620 raft_consensus.cc:504] T 00000000000000000000000000000000 P d2fe976495bf4db09c551bceed68ed96 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fe53874142bb48fc95f6360091185405" member_type: VOTER last_known_addr { host: "127.10.33.126" port: 37191 } } peers { permanent_uuid: "23670df4991843518252f60c82410195" member_type: VOTER last_known_addr { host: "127.10.33.125" port: 46227 } } peers { permanent_uuid: "d2fe976495bf4db09c551bceed68ed96" member_type: VOTER last_known_addr { host: "127.10.33.124" port: 36931 } }
11:44:56.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.055316 10428 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d2fe976495bf4db09c551bceed68ed96" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fe53874142bb48fc95f6360091185405" is_pre_election: true
11:44:56.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.056264 10428 raft_consensus.cc:2454] T 00000000000000000000000000000000 P fe53874142bb48fc95f6360091185405 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d2fe976495bf4db09c551bceed68ed96 in term 0.
11:44:56.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.056610 10620 leader_election.cc:288] T 00000000000000000000000000000000 P d2fe976495bf4db09c551bceed68ed96 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers fe53874142bb48fc95f6360091185405 (127.10.33.126:37191), 23670df4991843518252f60c82410195 (127.10.33.125:46227)
11:44:56.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.057942 10556 leader_election.cc:302] T 00000000000000000000000000000000 P d2fe976495bf4db09c551bceed68ed96 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: d2fe976495bf4db09c551bceed68ed96, fe53874142bb48fc95f6360091185405; no voters:
11:44:56.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.058923 10620 raft_consensus.cc:2781] T 00000000000000000000000000000000 P d2fe976495bf4db09c551bceed68ed96 [term 0 FOLLOWER]: Leader pre-election won for term 1
11:44:56.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.059715 10620 raft_consensus.cc:482] T 00000000000000000000000000000000 P d2fe976495bf4db09c551bceed68ed96 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
11:44:56.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.060974 10620 raft_consensus.cc:3037] T 00000000000000000000000000000000 P d2fe976495bf4db09c551bceed68ed96 [term 0 FOLLOWER]: Advancing to term 1
11:44:56.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.067428 10620 raft_consensus.cc:504] T 00000000000000000000000000000000 P d2fe976495bf4db09c551bceed68ed96 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fe53874142bb48fc95f6360091185405" member_type: VOTER last_known_addr { host: "127.10.33.126" port: 37191 } } peers { permanent_uuid: "23670df4991843518252f60c82410195" member_type: VOTER last_known_addr { host: "127.10.33.125" port: 46227 } } peers { permanent_uuid: "d2fe976495bf4db09c551bceed68ed96" member_type: VOTER last_known_addr { host: "127.10.33.124" port: 36931 } }
11:44:56.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.070843 10620 leader_election.cc:288] T 00000000000000000000000000000000 P d2fe976495bf4db09c551bceed68ed96 [CANDIDATE]: Term 1 election: Requested vote from peers fe53874142bb48fc95f6360091185405 (127.10.33.126:37191), 23670df4991843518252f60c82410195 (127.10.33.125:46227)
11:44:56.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.071908 10428 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d2fe976495bf4db09c551bceed68ed96" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fe53874142bb48fc95f6360091185405"
11:44:56.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.072453 10428 raft_consensus.cc:3037] T 00000000000000000000000000000000 P fe53874142bb48fc95f6360091185405 [term 0 FOLLOWER]: Advancing to term 1
11:44:56.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.079591 10428 raft_consensus.cc:2454] T 00000000000000000000000000000000 P fe53874142bb48fc95f6360091185405 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d2fe976495bf4db09c551bceed68ed96 in term 1.
11:44:56.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.085681 10556 leader_election.cc:302] T 00000000000000000000000000000000 P d2fe976495bf4db09c551bceed68ed96 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: d2fe976495bf4db09c551bceed68ed96, fe53874142bb48fc95f6360091185405; no voters:
11:44:56.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.086517 10620 raft_consensus.cc:2781] T 00000000000000000000000000000000 P d2fe976495bf4db09c551bceed68ed96 [term 1 FOLLOWER]: Leader election won for term 1
11:44:56.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.088835 10620 raft_consensus.cc:686] T 00000000000000000000000000000000 P d2fe976495bf4db09c551bceed68ed96 [term 1 LEADER]: Becoming Leader. State: Replica: d2fe976495bf4db09c551bceed68ed96, State: Running, Role: LEADER
11:44:56.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.089939 10620 consensus_queue.cc:237] T 00000000000000000000000000000000 P d2fe976495bf4db09c551bceed68ed96 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fe53874142bb48fc95f6360091185405" member_type: VOTER last_known_addr { host: "127.10.33.126" port: 37191 } } peers { permanent_uuid: "23670df4991843518252f60c82410195" member_type: VOTER last_known_addr { host: "127.10.33.125" port: 46227 } } peers { permanent_uuid: "d2fe976495bf4db09c551bceed68ed96" member_type: VOTER last_known_addr { host: "127.10.33.124" port: 36931 } }
11:44:56.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.097551 10646 sys_catalog.cc:455] T 00000000000000000000000000000000 P d2fe976495bf4db09c551bceed68ed96 [sys.catalog]: SysCatalogTable state changed. Reason: New leader d2fe976495bf4db09c551bceed68ed96. Latest consensus state: current_term: 1 leader_uuid: "d2fe976495bf4db09c551bceed68ed96" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fe53874142bb48fc95f6360091185405" member_type: VOTER last_known_addr { host: "127.10.33.126" port: 37191 } } peers { permanent_uuid: "23670df4991843518252f60c82410195" member_type: VOTER last_known_addr { host: "127.10.33.125" port: 46227 } } peers { permanent_uuid: "d2fe976495bf4db09c551bceed68ed96" member_type: VOTER last_known_addr { host: "127.10.33.124" port: 36931 } } }
11:44:56.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.098534 10646 sys_catalog.cc:458] T 00000000000000000000000000000000 P d2fe976495bf4db09c551bceed68ed96 [sys.catalog]: This master's current role is: LEADER
11:44:56.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.104940 10648 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
11:44:56.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.112118 10648 catalog_manager.cc:1484] Initializing Kudu cluster ID...
11:44:56.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.140857 10428 raft_consensus.cc:1261] T 00000000000000000000000000000000 P fe53874142bb48fc95f6360091185405 [term 1 FOLLOWER]: Refusing update from remote peer d2fe976495bf4db09c551bceed68ed96: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
11:44:56.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.145949 10646 consensus_queue.cc:1035] T 00000000000000000000000000000000 P d2fe976495bf4db09c551bceed68ed96 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fe53874142bb48fc95f6360091185405" member_type: VOTER last_known_addr { host: "127.10.33.126" port: 37191 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
11:44:56.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.190263 10622 sys_catalog.cc:455] T 00000000000000000000000000000000 P fe53874142bb48fc95f6360091185405 [sys.catalog]: SysCatalogTable state changed. Reason: New leader d2fe976495bf4db09c551bceed68ed96. Latest consensus state: current_term: 1 leader_uuid: "d2fe976495bf4db09c551bceed68ed96" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fe53874142bb48fc95f6360091185405" member_type: VOTER last_known_addr { host: "127.10.33.126" port: 37191 } } peers { permanent_uuid: "23670df4991843518252f60c82410195" member_type: VOTER last_known_addr { host: "127.10.33.125" port: 46227 } } peers { permanent_uuid: "d2fe976495bf4db09c551bceed68ed96" member_type: VOTER last_known_addr { host: "127.10.33.124" port: 36931 } } }
11:44:56.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.193287 10622 sys_catalog.cc:458] T 00000000000000000000000000000000 P fe53874142bb48fc95f6360091185405 [sys.catalog]: This master's current role is: FOLLOWER
11:44:56.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.194355 10649 mvcc.cc:205] Tried to move back new op lower bound from 7334634684967297024 to 7334634684794343424. Current Snapshot: MvccSnapshot[applied={T|T < 7334634684967297024}]
11:44:56.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.199680 10650 mvcc.cc:205] Tried to move back new op lower bound from 7334634684967297024 to 7334634684794343424. Current Snapshot: MvccSnapshot[applied={T|T < 7334634684967297024}]
11:44:56.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.204836 10620 sys_catalog.cc:455] T 00000000000000000000000000000000 P d2fe976495bf4db09c551bceed68ed96 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "d2fe976495bf4db09c551bceed68ed96" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fe53874142bb48fc95f6360091185405" member_type: VOTER last_known_addr { host: "127.10.33.126" port: 37191 } } peers { permanent_uuid: "23670df4991843518252f60c82410195" member_type: VOTER last_known_addr { host: "127.10.33.125" port: 46227 } } peers { permanent_uuid: "d2fe976495bf4db09c551bceed68ed96" member_type: VOTER last_known_addr { host: "127.10.33.124" port: 36931 } } }
11:44:56.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.205523 10620 sys_catalog.cc:458] T 00000000000000000000000000000000 P d2fe976495bf4db09c551bceed68ed96 [sys.catalog]: This master's current role is: LEADER
11:44:56.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.209335 10648 catalog_manager.cc:1347] Generated new cluster ID: 25331014b6eb4c7592b3a24f0dc27370
11:44:56.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.209623 10648 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
11:44:56.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.217303 10622 sys_catalog.cc:455] T 00000000000000000000000000000000 P fe53874142bb48fc95f6360091185405 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "d2fe976495bf4db09c551bceed68ed96" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fe53874142bb48fc95f6360091185405" member_type: VOTER last_known_addr { host: "127.10.33.126" port: 37191 } } peers { permanent_uuid: "23670df4991843518252f60c82410195" member_type: VOTER last_known_addr { host: "127.10.33.125" port: 46227 } } peers { permanent_uuid: "d2fe976495bf4db09c551bceed68ed96" member_type: VOTER last_known_addr { host: "127.10.33.124" port: 36931 } } }
11:44:56.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.218039 10622 sys_catalog.cc:458] T 00000000000000000000000000000000 P fe53874142bb48fc95f6360091185405 [sys.catalog]: This master's current role is: FOLLOWER
11:44:56.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.245731 10648 catalog_manager.cc:1370] Generated new certificate authority record
11:44:56.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.248237 10648 catalog_manager.cc:1504] Loading token signing keys...
11:44:56.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.301924 10648 catalog_manager.cc:5868] T 00000000000000000000000000000000 P d2fe976495bf4db09c551bceed68ed96: Generated new TSK 0
11:44:56.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.311846 10648 catalog_manager.cc:1514] Initializing in-progress tserver states...
11:44:56.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.349089 10373 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskst9njx/build/tsan/bin/kudu
11:44:56.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskst9njx/build/tsan/bin/kudu
11:44:56.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/ts-0/wal
11:44:56.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/ts-0/data
11:44:56.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:44:56.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:44:56.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:44:56.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:44:56.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:44:56.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:44:56.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/ts-0/logs
11:44:56.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/ts-0/data/info.pb
11:44:56.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:44:56.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:44:56.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:44:56.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:44:56.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:44:56.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:44:56.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:44:56.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:44:56.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
11:44:56.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:44:56.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.33.65:0
11:44:56.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.33.65
11:44:56.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.33.65
11:44:56.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:44:56.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.33.126:37191,127.10.33.125:46227,127.10.33.124:36931
11:44:56.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.33.84:35767
11:44:56.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:44:56.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:44:56.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
11:44:56.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.366243 10528 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.33.124" port: 36931 } attempt: 7
11:44:56.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.419859 10528 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 23670df4991843518252f60c82410195: Bootstrap starting.
11:44:56.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.437994 10528 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 23670df4991843518252f60c82410195: Neither blocks nor log segments found. Creating new log.
11:44:56.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.444597 10528 log.cc:824] T 00000000000000000000000000000000 P 23670df4991843518252f60c82410195: Log is configured to *not* fsync() on all Append() calls
11:44:56.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.466313 10528 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 23670df4991843518252f60c82410195: No bootstrap required, opened a new log
11:44:56.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.543286 10528 raft_consensus.cc:348] T 00000000000000000000000000000000 P 23670df4991843518252f60c82410195 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fe53874142bb48fc95f6360091185405" member_type: VOTER last_known_addr { host: "127.10.33.126" port: 37191 } } peers { permanent_uuid: "23670df4991843518252f60c82410195" member_type: VOTER last_known_addr { host: "127.10.33.125" port: 46227 } } peers { permanent_uuid: "d2fe976495bf4db09c551bceed68ed96" member_type: VOTER last_known_addr { host: "127.10.33.124" port: 36931 } }
11:44:56.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.545701 10528 raft_consensus.cc:374] T 00000000000000000000000000000000 P 23670df4991843518252f60c82410195 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
11:44:56.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.546571 10528 raft_consensus.cc:729] T 00000000000000000000000000000000 P 23670df4991843518252f60c82410195 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 23670df4991843518252f60c82410195, State: Initialized, Role: FOLLOWER
11:44:56.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.548911 10528 consensus_queue.cc:260] T 00000000000000000000000000000000 P 23670df4991843518252f60c82410195 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fe53874142bb48fc95f6360091185405" member_type: VOTER last_known_addr { host: "127.10.33.126" port: 37191 } } peers { permanent_uuid: "23670df4991843518252f60c82410195" member_type: VOTER last_known_addr { host: "127.10.33.125" port: 46227 } } peers { permanent_uuid: "d2fe976495bf4db09c551bceed68ed96" member_type: VOTER last_known_addr { host: "127.10.33.124" port: 36931 } }
11:44:56.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.572531 10502 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d2fe976495bf4db09c551bceed68ed96" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "23670df4991843518252f60c82410195"
11:44:56.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.573307 10502 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 23670df4991843518252f60c82410195 [term 0 FOLLOWER]: Advancing to term 1
11:44:56.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.577366 10503 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d2fe976495bf4db09c551bceed68ed96" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "23670df4991843518252f60c82410195" is_pre_election: true
11:44:56.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.584195 10502 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 23670df4991843518252f60c82410195 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d2fe976495bf4db09c551bceed68ed96 in term 1.
11:44:56.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.585481 10501 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 23670df4991843518252f60c82410195 [term 1 FOLLOWER]: Refusing update from remote peer d2fe976495bf4db09c551bceed68ed96: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
11:44:56.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.591813 10646 consensus_queue.cc:1035] T 00000000000000000000000000000000 P d2fe976495bf4db09c551bceed68ed96 [LEADER]: Connected to new peer: Peer: permanent_uuid: "23670df4991843518252f60c82410195" member_type: VOTER last_known_addr { host: "127.10.33.125" port: 46227 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
11:44:56.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.601024 10658 sys_catalog.cc:455] T 00000000000000000000000000000000 P 23670df4991843518252f60c82410195 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fe53874142bb48fc95f6360091185405" member_type: VOTER last_known_addr { host: "127.10.33.126" port: 37191 } } peers { permanent_uuid: "23670df4991843518252f60c82410195" member_type: VOTER last_known_addr { host: "127.10.33.125" port: 46227 } } peers { permanent_uuid: "d2fe976495bf4db09c551bceed68ed96" member_type: VOTER last_known_addr { host: "127.10.33.124" port: 36931 } } }
11:44:56.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.603302 10658 sys_catalog.cc:458] T 00000000000000000000000000000000 P 23670df4991843518252f60c82410195 [sys.catalog]: This master's current role is: FOLLOWER
11:44:56.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.602097 10528 sys_catalog.cc:564] T 00000000000000000000000000000000 P 23670df4991843518252f60c82410195 [sys.catalog]: configured and running, proceeding with master startup.
11:44:56.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.648645 10661 mvcc.cc:205] Tried to move back new op lower bound from 7334634684967297024 to 7334634684794343424. Current Snapshot: MvccSnapshot[applied={T|T < 7334634684967297024}]
11:44:56.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.676317 10659 sys_catalog.cc:455] T 00000000000000000000000000000000 P 23670df4991843518252f60c82410195 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "d2fe976495bf4db09c551bceed68ed96" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fe53874142bb48fc95f6360091185405" member_type: VOTER last_known_addr { host: "127.10.33.126" port: 37191 } } peers { permanent_uuid: "23670df4991843518252f60c82410195" member_type: VOTER last_known_addr { host: "127.10.33.125" port: 46227 } } peers { permanent_uuid: "d2fe976495bf4db09c551bceed68ed96" member_type: VOTER last_known_addr { host: "127.10.33.124" port: 36931 } } }
11:44:56.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.677071 10659 sys_catalog.cc:458] T 00000000000000000000000000000000 P 23670df4991843518252f60c82410195 [sys.catalog]: This master's current role is: FOLLOWER
11:44:56.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.675000 10658 sys_catalog.cc:455] T 00000000000000000000000000000000 P 23670df4991843518252f60c82410195 [sys.catalog]: SysCatalogTable state changed. Reason: New leader d2fe976495bf4db09c551bceed68ed96. Latest consensus state: current_term: 1 leader_uuid: "d2fe976495bf4db09c551bceed68ed96" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fe53874142bb48fc95f6360091185405" member_type: VOTER last_known_addr { host: "127.10.33.126" port: 37191 } } peers { permanent_uuid: "23670df4991843518252f60c82410195" member_type: VOTER last_known_addr { host: "127.10.33.125" port: 46227 } } peers { permanent_uuid: "d2fe976495bf4db09c551bceed68ed96" member_type: VOTER last_known_addr { host: "127.10.33.124" port: 36931 } } }
11:44:56.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.678061 10658 sys_catalog.cc:458] T 00000000000000000000000000000000 P 23670df4991843518252f60c82410195 [sys.catalog]: This master's current role is: FOLLOWER
11:44:56.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.714039 10674 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
11:44:56.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:56.798516 10654 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:44:56.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:56.799566 10654 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:44:56.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:56.800542 10654 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:44:56.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:56.856724 10654 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:44:56.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:56.858312 10654 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.33.65
11:44:56.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.894126 10654 tablet_server_runner.cc:78] Tablet server non-default flags:
11:44:56.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:44:56.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.33.84:35767
11:44:56.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:44:56.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:44:56.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/ts-0/data
11:44:56.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/ts-0/wal
11:44:56.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:44:56.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:44:56.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.33.65:0
11:44:56.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:44:56.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:44:56.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:44:56.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/ts-0/data/info.pb
11:44:56.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.33.65
11:44:56.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:44:56.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.33.126:37191,127.10.33.125:46227,127.10.33.124:36931
11:44:56.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:44:56.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:44:56.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:44:56.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:44:56.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:44:56.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.33.65
11:44:56.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/ts-0/logs
11:44:56.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:44:56.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:44:56.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:44:56.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
11:44:56.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:44:56.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:44:56.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:44:56.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 10:52:50 UTC on ccycloud.quasar-peqjre.root.comops.site
11:44:56.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679900
11:44:56.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:44:56.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.895593 10654 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:44:56.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.897416 10654 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:44:56.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:56.922129 10681 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:44:56.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.987232 10644 catalog_manager.cc:1259] Loaded cluster ID: 25331014b6eb4c7592b3a24f0dc27370
11:44:56.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.987504 10644 catalog_manager.cc:1552] T 00000000000000000000000000000000 P fe53874142bb48fc95f6360091185405: loading cluster ID for follower catalog manager: success
11:44:56.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:56.994983 10644 catalog_manager.cc:1574] T 00000000000000000000000000000000 P fe53874142bb48fc95f6360091185405: acquiring CA information for follower catalog manager: success
11:44:57.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:57.008504 10644 catalog_manager.cc:1602] T 00000000000000000000000000000000 P fe53874142bb48fc95f6360091185405: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
11:44:57.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:57.011663 10646 sys_catalog.cc:455] T 00000000000000000000000000000000 P d2fe976495bf4db09c551bceed68ed96 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "d2fe976495bf4db09c551bceed68ed96" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fe53874142bb48fc95f6360091185405" member_type: VOTER last_known_addr { host: "127.10.33.126" port: 37191 } } peers { permanent_uuid: "23670df4991843518252f60c82410195" member_type: VOTER last_known_addr { host: "127.10.33.125" port: 46227 } } peers { permanent_uuid: "d2fe976495bf4db09c551bceed68ed96" member_type: VOTER last_known_addr { host: "127.10.33.124" port: 36931 } } }
11:44:57.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:57.012949 10646 sys_catalog.cc:458] T 00000000000000000000000000000000 P d2fe976495bf4db09c551bceed68ed96 [sys.catalog]: This master's current role is: LEADER
11:44:57.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:57.740514 10674 catalog_manager.cc:1259] Loaded cluster ID: 25331014b6eb4c7592b3a24f0dc27370
11:44:57.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:57.740785 10674 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 23670df4991843518252f60c82410195: loading cluster ID for follower catalog manager: success
11:44:57.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:57.760639 10674 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 23670df4991843518252f60c82410195: acquiring CA information for follower catalog manager: success
11:44:57.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:57.766425 10674 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 23670df4991843518252f60c82410195: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
11:44:58.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:56.928622 10680 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:44:58.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:56.927073 10684 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:44:58.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:58.934418 10682 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2011 milliseconds
11:44:58.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:58.934536 10654 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
11:44:58.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:58.935840 10654 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:44:58.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:58.947602 10654 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:44:58.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:58.949210 10654 hybrid_clock.cc:648] HybridClock initialized: now 1790682298949150 us; error 89 us; skew 500 ppm
11:44:58.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:58.950418 10654 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:44:58.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:58.977001 10654 webserver.cc:460] Webserver started at http://127.10.33.65:33295/ using document root <none> and password file <none>
11:44:58.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:58.978557 10654 fs_manager.cc:359] Metadata directory not provided
11:44:58.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:58.979007 10654 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:44:58.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:58.979750 10654 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
11:44:58.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:58.989084 10654 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/ts-0/data/instance:
11:44:58.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d1f5bf465a394ab1a10b076adb7a4fc1"
11:44:58.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:44:58 on dist-test-slave-8d15"
11:44:58.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:58.992882 10654 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/ts-0/wal/instance:
11:44:58.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d1f5bf465a394ab1a10b076adb7a4fc1"
11:44:58.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:44:58 on dist-test-slave-8d15"
11:44:59.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:59.009052 10654 fs_manager.cc:691] Time spent creating directory manager: real 0.015s user 0.009s sys 0.004s
11:44:59.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:59.022380 10693 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:44:59.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:59.023952 10654 fs_manager.cc:725] Time spent opening block manager: real 0.010s user 0.005s sys 0.004s
11:44:59.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:59.024498 10654 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/ts-0/data,/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/ts-0/wal
11:44:59.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d1f5bf465a394ab1a10b076adb7a4fc1"
11:44:59.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:44:58 on dist-test-slave-8d15"
11:44:59.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:59.025173 10654 fs_report.cc:383] FS layout report
11:44:59.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:44:59.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/ts-0/wal
11:44:59.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/ts-0/wal
11:44:59.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/ts-0/data/data
11:44:59.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:44:59.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:44:59.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:44:59.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:44:59.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:44:59.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:44:59.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:44:59.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:44:59.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:44:59.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:44:59.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:44:59.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:59.086720 10654 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:44:59.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:59.088162 10654 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:44:59.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:59.088564 10654 kserver.cc:163] Server-wide thread pool size limit: 3276
11:44:59.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:59.091001 10654 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
11:44:59.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:59.095458 10654 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
11:44:59.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:59.095738 10654 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.001s sys 0.000s
11:44:59.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:59.096069 10654 ts_tablet_manager.cc:610] Registered 0 tablets
11:44:59.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:59.096256 10654 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
11:44:59.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:59.275297 10654 rpc_server.cc:307] RPC server started. Bound to: 127.10.33.65:40483
11:44:59.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:59.275416 10805 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.33.65:40483 every 8 connection(s)
11:44:59.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:59.279027 10654 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/ts-0/data/info.pb
11:44:59.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:59.288223 10373 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskst9njx/build/tsan/bin/kudu as pid 10654
11:44:59.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:59.288650 10373 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/ts-0/wal/instance
11:44:59.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:59.308758 10373 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskst9njx/build/tsan/bin/kudu
11:44:59.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskst9njx/build/tsan/bin/kudu
11:44:59.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/ts-1/wal
11:44:59.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/ts-1/data
11:44:59.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:44:59.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:44:59.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:44:59.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:44:59.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:44:59.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:44:59.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/ts-1/logs
11:44:59.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/ts-1/data/info.pb
11:44:59.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:44:59.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:44:59.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:44:59.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:44:59.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:44:59.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:44:59.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:44:59.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:44:59.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
11:44:59.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:44:59.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.33.66:0
11:44:59.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.33.66
11:44:59.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.33.66
11:44:59.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:44:59.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.33.126:37191,127.10.33.125:46227,127.10.33.124:36931
11:44:59.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.33.84:35767
11:44:59.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:44:59.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:44:59.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
11:44:59.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:59.358896 10807 heartbeater.cc:347] Connected to a master server at 127.10.33.125:46227
11:44:59.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:59.359372 10807 heartbeater.cc:464] Registering TS with master...
11:44:59.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:59.362129 10807 heartbeater.cc:511] Master 127.10.33.125:46227 requested a full tablet report, sending...
11:44:59.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:59.362447 10806 heartbeater.cc:347] Connected to a master server at 127.10.33.124:36931
11:44:59.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:59.362805 10806 heartbeater.cc:464] Registering TS with master...
11:44:59.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:59.363606 10806 heartbeater.cc:511] Master 127.10.33.124:36931 requested a full tablet report, sending...
11:44:59.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:59.365648 10493 ts_manager.cc:194] Registered new tserver with Master: d1f5bf465a394ab1a10b076adb7a4fc1 (127.10.33.65:40483)
11:44:59.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:59.367977 10572 ts_manager.cc:194] Registered new tserver with Master: d1f5bf465a394ab1a10b076adb7a4fc1 (127.10.33.65:40483)
11:44:59.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:59.369102 10812 heartbeater.cc:347] Connected to a master server at 127.10.33.126:37191
11:44:59.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:59.369428 10812 heartbeater.cc:464] Registering TS with master...
11:44:59.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:59.370146 10812 heartbeater.cc:511] Master 127.10.33.126:37191 requested a full tablet report, sending...
11:44:59.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:59.370765 10572 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.33.65:45395
11:44:59.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:59.372973 10418 ts_manager.cc:194] Registered new tserver with Master: d1f5bf465a394ab1a10b076adb7a4fc1 (127.10.33.65:40483)
11:44:59.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:59.684855 10813 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:44:59.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:59.685432 10813 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:44:59.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:59.686035 10813 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:44:59.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:59.717357 10813 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:44:59.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:59.718290 10813 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.33.66
11:44:59.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:59.752136 10813 tablet_server_runner.cc:78] Tablet server non-default flags:
11:44:59.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:44:59.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.33.84:35767
11:44:59.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:44:59.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:44:59.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/ts-1/data
11:44:59.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/ts-1/wal
11:44:59.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:44:59.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:44:59.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.33.66:0
11:44:59.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:44:59.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:44:59.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:44:59.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/ts-1/data/info.pb
11:44:59.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.33.66
11:44:59.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:44:59.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.33.126:37191,127.10.33.125:46227,127.10.33.124:36931
11:44:59.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:44:59.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:44:59.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:44:59.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:44:59.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:44:59.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.33.66
11:44:59.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/ts-1/logs
11:44:59.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:44:59.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:44:59.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:44:59.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
11:44:59.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:44:59.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:44:59.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:44:59.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 10:52:50 UTC on ccycloud.quasar-peqjre.root.comops.site
11:44:59.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679900
11:44:59.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:44:59.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:59.753629 10813 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:44:59.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:44:59.755533 10813 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:44:59.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:59.782536 10823 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:45:00.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:00.375190 10806 heartbeater.cc:503] Master 127.10.33.124:36931 was elected leader, sending a full tablet report...
11:45:01.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:59.783919 10822 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:45:01.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:44:59.788692 10825 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:45:01.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:01.015218 10824 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
11:45:01.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:01.015681 10813 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
11:45:01.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:01.019819 10813 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:45:01.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:01.022431 10813 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:45:01.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:01.023840 10813 hybrid_clock.cc:648] HybridClock initialized: now 1790682301023711 us; error 124 us; skew 500 ppm
11:45:01.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:01.024809 10813 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:45:01.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:01.032136 10813 webserver.cc:460] Webserver started at http://127.10.33.66:34593/ using document root <none> and password file <none>
11:45:01.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:01.033218 10813 fs_manager.cc:359] Metadata directory not provided
11:45:01.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:01.033207 10449 debug-util.cc:398] Leaking SignalData structure 0x7b0800067fc0 after lost signal to thread 10386
11:45:01.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:01.033457 10813 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:45:01.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:01.033954 10813 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
11:45:01.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:01.034057 10449 debug-util.cc:398] Leaking SignalData structure 0x7b08000a51a0 after lost signal to thread 10452
11:45:01.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:01.040599 10813 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/ts-1/data/instance:
11:45:01.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f7792b6d48794884b62c5dcc30ed7ca2"
11:45:01.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:45:01 on dist-test-slave-8d15"
11:45:01.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:01.041939 10813 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/ts-1/wal/instance:
11:45:01.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f7792b6d48794884b62c5dcc30ed7ca2"
11:45:01.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:45:01 on dist-test-slave-8d15"
11:45:01.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:01.051134 10813 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.010s sys 0.001s
11:45:01.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:01.058007 10834 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:45:01.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:01.059082 10813 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.001s
11:45:01.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:01.059345 10813 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/ts-1/data,/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/ts-1/wal
11:45:01.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f7792b6d48794884b62c5dcc30ed7ca2"
11:45:01.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:45:01 on dist-test-slave-8d15"
11:45:01.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:01.059701 10813 fs_report.cc:383] FS layout report
11:45:01.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:45:01.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/ts-1/wal
11:45:01.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/ts-1/wal
11:45:01.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/ts-1/data/data
11:45:01.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:45:01.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:45:01.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:45:01.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:45:01.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:45:01.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:45:01.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:45:01.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:45:01.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:45:01.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:45:01.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:45:01.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:01.105739 10813 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:45:01.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:01.107175 10813 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:45:01.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:01.107537 10813 kserver.cc:163] Server-wide thread pool size limit: 3276
11:45:01.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:01.110143 10813 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
11:45:01.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:01.114984 10813 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
11:45:01.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:01.115173 10813 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
11:45:01.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:01.115594 10813 ts_tablet_manager.cc:610] Registered 0 tablets
11:45:01.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:01.115818 10813 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
11:45:01.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:01.306805 10813 rpc_server.cc:307] RPC server started. Bound to: 127.10.33.66:34427
11:45:01.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:01.306891 10946 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.33.66:34427 every 8 connection(s)
11:45:01.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:01.309425 10813 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/ts-1/data/info.pb
11:45:01.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:01.312702 10373 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskst9njx/build/tsan/bin/kudu as pid 10813
11:45:01.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:01.313335 10373 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/ts-1/wal/instance
11:45:01.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:01.328804 10373 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskst9njx/build/tsan/bin/kudu
11:45:01.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskst9njx/build/tsan/bin/kudu
11:45:01.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/ts-2/wal
11:45:01.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/ts-2/data
11:45:01.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:45:01.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:45:01.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:45:01.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:45:01.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:45:01.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:45:01.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/ts-2/logs
11:45:01.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/ts-2/data/info.pb
11:45:01.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:45:01.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:45:01.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:45:01.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:45:01.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:45:01.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:45:01.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:45:01.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:45:01.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
11:45:01.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:45:01.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.33.67:0
11:45:01.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.33.67
11:45:01.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.33.67
11:45:01.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:45:01.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.33.126:37191,127.10.33.125:46227,127.10.33.124:36931
11:45:01.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.33.84:35767
11:45:01.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:45:01.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:45:01.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
11:45:01.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:01.389526 10948 heartbeater.cc:347] Connected to a master server at 127.10.33.125:46227
11:45:01.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:01.390399 10948 heartbeater.cc:464] Registering TS with master...
11:45:01.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:01.391609 10948 heartbeater.cc:511] Master 127.10.33.125:46227 requested a full tablet report, sending...
11:45:01.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:01.394271 10493 ts_manager.cc:194] Registered new tserver with Master: f7792b6d48794884b62c5dcc30ed7ca2 (127.10.33.66:34427)
11:45:01.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:01.408703 10947 heartbeater.cc:347] Connected to a master server at 127.10.33.124:36931
11:45:01.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:01.409003 10947 heartbeater.cc:464] Registering TS with master...
11:45:01.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:01.409682 10947 heartbeater.cc:511] Master 127.10.33.124:36931 requested a full tablet report, sending...
11:45:01.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:01.411521 10572 ts_manager.cc:194] Registered new tserver with Master: f7792b6d48794884b62c5dcc30ed7ca2 (127.10.33.66:34427)
11:45:01.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:01.413477 10572 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.33.66:34949
11:45:01.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:01.422230 10951 heartbeater.cc:347] Connected to a master server at 127.10.33.126:37191
11:45:01.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:01.422542 10951 heartbeater.cc:464] Registering TS with master...
11:45:01.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:01.423126 10951 heartbeater.cc:511] Master 127.10.33.126:37191 requested a full tablet report, sending...
11:45:01.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:01.425722 10418 ts_manager.cc:194] Registered new tserver with Master: f7792b6d48794884b62c5dcc30ed7ca2 (127.10.33.66:34427)
11:45:01.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:01.710649 10954 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:45:01.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:01.711191 10954 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:45:01.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:01.711782 10954 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:45:01.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:01.742678 10954 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:45:01.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:01.743680 10954 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.33.67
11:45:01.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:01.780491 10954 tablet_server_runner.cc:78] Tablet server non-default flags:
11:45:01.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:45:01.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.33.84:35767
11:45:01.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:45:01.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:45:01.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/ts-2/data
11:45:01.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/ts-2/wal
11:45:01.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:45:01.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:45:01.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.33.67:0
11:45:01.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:45:01.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:45:01.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:45:01.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/ts-2/data/info.pb
11:45:01.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.33.67
11:45:01.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:45:01.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.33.126:37191,127.10.33.125:46227,127.10.33.124:36931
11:45:01.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:45:01.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:45:01.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:45:01.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:45:01.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:45:01.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.33.67
11:45:01.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/ts-2/logs
11:45:01.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:45:01.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:45:01.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:45:01.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
11:45:01.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:45:01.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:45:01.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:45:01.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 10:52:50 UTC on ccycloud.quasar-peqjre.root.comops.site
11:45:01.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679900
11:45:01.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:45:01.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:01.781981 10954 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:45:01.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:01.784252 10954 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:45:01.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:01.808514 10963 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:45:02.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:02.418040 10947 heartbeater.cc:503] Master 127.10.33.124:36931 was elected leader, sending a full tablet report...
11:45:03.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:01.816149 10964 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:45:03.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:01.817962 10966 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:45:03.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:03.038003 10965 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
11:45:03.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:03.038057 10954 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
11:45:03.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:03.041764 10954 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:45:03.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:03.043912 10954 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:45:03.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:03.045320 10954 hybrid_clock.cc:648] HybridClock initialized: now 1790682303045297 us; error 63 us; skew 500 ppm
11:45:03.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:03.046049 10954 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:45:03.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:03.052537 10954 webserver.cc:460] Webserver started at http://127.10.33.67:37041/ using document root <none> and password file <none>
11:45:03.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:03.053337 10954 fs_manager.cc:359] Metadata directory not provided
11:45:03.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:03.053506 10954 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:45:03.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:03.053857 10954 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
11:45:03.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:03.058400 10954 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/ts-2/data/instance:
11:45:03.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aabe1bb7f0874eb99e1c131f00599d46"
11:45:03.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:45:03 on dist-test-slave-8d15"
11:45:03.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:03.059386 10954 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/ts-2/wal/instance:
11:45:03.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aabe1bb7f0874eb99e1c131f00599d46"
11:45:03.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:45:03 on dist-test-slave-8d15"
11:45:03.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:03.065986 10954 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.005s sys 0.000s
11:45:03.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:03.071388 10974 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:45:03.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:03.072531 10954 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.001s sys 0.002s
11:45:03.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:03.072793 10954 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/ts-2/data,/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/ts-2/wal
11:45:03.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aabe1bb7f0874eb99e1c131f00599d46"
11:45:03.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:45:03 on dist-test-slave-8d15"
11:45:03.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:03.073103 10954 fs_report.cc:383] FS layout report
11:45:03.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:45:03.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/ts-2/wal
11:45:03.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/ts-2/wal
11:45:03.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/ts-2/data/data
11:45:03.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:45:03.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:45:03.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:45:03.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:45:03.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:45:03.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:45:03.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:45:03.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:45:03.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:45:03.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:45:03.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:45:03.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:03.117146 10954 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:45:03.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:03.118410 10954 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:45:03.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:03.118853 10954 kserver.cc:163] Server-wide thread pool size limit: 3276
11:45:03.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:03.121263 10954 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
11:45:03.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:03.125264 10954 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
11:45:03.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:03.125553 10954 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
11:45:03.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:03.125756 10954 ts_tablet_manager.cc:610] Registered 0 tablets
11:45:03.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:03.125910 10954 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
11:45:03.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:03.272531 10954 rpc_server.cc:307] RPC server started. Bound to: 127.10.33.67:39227
11:45:03.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:03.272631 11086 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.33.67:39227 every 8 connection(s)
11:45:03.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:03.275803 10954 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/ts-2/data/info.pb
11:45:03.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:03.279170 10373 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskst9njx/build/tsan/bin/kudu as pid 10954
11:45:03.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:03.279668 10373 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster3320167242087921590/ts-2/wal/instance
11:45:03.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:03.349304 11087 heartbeater.cc:347] Connected to a master server at 127.10.33.124:36931
11:45:03.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:03.349823 11087 heartbeater.cc:464] Registering TS with master...
11:45:03.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:03.350875 11087 heartbeater.cc:511] Master 127.10.33.124:36931 requested a full tablet report, sending...
11:45:03.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:03.353760 11088 heartbeater.cc:347] Connected to a master server at 127.10.33.125:46227
11:45:03.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:03.354056 11091 heartbeater.cc:347] Connected to a master server at 127.10.33.126:37191
11:45:03.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:03.354082 11088 heartbeater.cc:464] Registering TS with master...
11:45:03.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:03.354172 10572 ts_manager.cc:194] Registered new tserver with Master: aabe1bb7f0874eb99e1c131f00599d46 (127.10.33.67:39227)
11:45:03.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:03.354444 11091 heartbeater.cc:464] Registering TS with master...
11:45:03.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:03.354908 11088 heartbeater.cc:511] Master 127.10.33.125:46227 requested a full tablet report, sending...
11:45:03.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:03.355162 11091 heartbeater.cc:511] Master 127.10.33.126:37191 requested a full tablet report, sending...
11:45:03.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:03.356448 10572 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.33.67:56479
11:45:03.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:03.357270 10493 ts_manager.cc:194] Registered new tserver with Master: aabe1bb7f0874eb99e1c131f00599d46 (127.10.33.67:39227)
11:45:03.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:03.357800 10417 ts_manager.cc:194] Registered new tserver with Master: aabe1bb7f0874eb99e1c131f00599d46 (127.10.33.67:39227)
11:45:03.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:03.374480 10373 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
11:45:03.385 [DEBUG - main] (MiniKuduCluster.java:195) Response:
11:45:03.386 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
11:45:03.398 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.10.33.126"
port: 37191
}
bound_http_address {
host: "127.10.33.126"
port: 35907
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.10.33.125"
port: 46227
}
bound_http_address {
host: "127.10.33.125"
port: 33541
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.10.33.124"
port: 36931
}
bound_http_address {
host: "127.10.33.124"
port: 42815
}
}
}
11:45:03.403 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
11:45:03.406 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.10.33.65"
port: 40483
}
bound_http_address {
host: "127.10.33.65"
port: 33295
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.10.33.66"
port: 34427
}
bound_http_address {
host: "127.10.33.66"
port: 34593
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.10.33.67"
port: 39227
}
bound_http_address {
host: "127.10.33.67"
port: 37041
}
}
}
11:45:03.406 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
11:45:03.566 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
11:45:03.666 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskst9njx/test-tmp
11:45:03.666 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
11:45:03.667 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskst9njx/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
11:45:03.670 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
11:45:04.012 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:45:04.012 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
11:45:04.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:04.016494 11098 mini_oidc.cc:152] Starting JWKS server
11:45:04.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:04.020243 11098 webserver.cc:460] Webserver started at http://127.0.0.1:45657/ using document root <none> and password file <none>
11:45:04.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:04.020718 11098 mini_oidc.cc:189] Starting OIDC Discovery server
11:45:04.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:04.022899 11098 webserver.cc:460] Webserver started at http://127.0.0.1:43559/ using document root <none> and password file <none>
11:45:04.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
11:45:04.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/krb5kdc/principal' for realm 'KRBTEST.COM',
11:45:04.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
11:45:04.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:45:04 dist-test-slave-8d15 krb5kdc[11110](info): setting up network...
11:45:04.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
11:45:04.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:45:04 dist-test-slave-8d15 krb5kdc[11110](info): set up 2 sockets
11:45:04.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:45:04 dist-test-slave-8d15 krb5kdc[11110](info): commencing operation
11:45:04.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
11:45:04.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:04.360659 11087 heartbeater.cc:503] Master 127.10.33.124:36931 was elected leader, sending a full tablet report...
11:45:05.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:05.131187 11098 mini_kdc.cc:121] Time spent starting KDC: real 1.084s user 0.005s sys 0.015s
11:45:05.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
11:45:05.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
11:45:05.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
11:45:05.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
11:45:05.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
11:45:05.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
11:45:05.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
11:45:05.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
11:45:05.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
11:45:05.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
11:45:05.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/krb5kdc/test-user.keytab.
11:45:05.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/krb5kdc/test-user.keytab.
11:45:05.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:45:05 dist-test-slave-8d15 krb5kdc[11110](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682305, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:45:05.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
11:45:05.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T11:45:05Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
11:45:05.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T11:45:05Z Disabled control of system clock
11:45:05.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.10.214.190@KRBTEST.COM; defaulting to no policy
11:45:05.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
11:45:05.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.10.214.190@KRBTEST.COM" created.
11:45:05.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
11:45:05.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.10.214.190 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/krb5kdc/kudu_127.10.214.190.keytab.
11:45:05.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.10.214.190 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/krb5kdc/kudu_127.10.214.190.keytab.
11:45:05.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:05.313709 11098 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskst9njx/build/tsan/bin/kudu
11:45:05.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskst9njx/build/tsan/bin/kudu
11:45:05.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-0/wal
11:45:05.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-0/data
11:45:05.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:45:05.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:45:05.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:45:05.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:45:05.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:45:05.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:45:05.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-0/logs
11:45:05.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-0/data/info.pb
11:45:05.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:45:05.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:45:05.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:45:05.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:45:05.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:45:05.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:45:05.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:45:05.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:45:05.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
11:45:05.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:45:05.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:45:05.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:45:05.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.214.190:41139
11:45:05.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.214.190
11:45:05.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:45:05.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/krb5kdc/kudu_127.10.214.190.keytab
11:45:05.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.10.214.190
11:45:05.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:45:05.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
11:45:05.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
11:45:05.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.214.148:40277
11:45:05.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:45:05.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:45:05.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:45:05.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:45:05.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.214.190:41139,127.10.214.189:36301,127.10.214.188:40593
11:45:05.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:45:05.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45657/jwks.json
11:45:05.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
11:45:05.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:05.618573 11125 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:45:05.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:05.619081 11125 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:45:05.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:05.619719 11125 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:45:05.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:05.652396 11125 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
11:45:05.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:05.652648 11125 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
11:45:05.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:05.653108 11125 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:45:05.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:05.653447 11125 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
11:45:05.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:05.653710 11125 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
11:45:05.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:05.653898 11125 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
11:45:05.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:05.654032 11125 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45657/jwks.json
11:45:05.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:05.691776 11125 master_runner.cc:386] Master server non-default flags:
11:45:05.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:45:05.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.214.148:40277
11:45:05.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:45:05.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:45:05.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-0/data
11:45:05.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-0/wal
11:45:05.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
11:45:05.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:45:05.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.214.190:41139,127.10.214.189:36301,127.10.214.188:40593
11:45:05.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/krb5kdc/kudu_127.10.214.190.keytab
11:45:05.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.10.214.190
11:45:05.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:45:05.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:45:05.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:45:05.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.214.190:41139
11:45:05.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:45:05.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:45:05.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:45:05.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45657/jwks.json
11:45:05.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:45:05.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
11:45:05.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
11:45:05.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:45:05.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:45:05.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-0/data/info.pb
11:45:05.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.214.190
11:45:05.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:45:05.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:45:05.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:45:05.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:45:05.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:45:05.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:45:05.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-0/logs
11:45:05.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:45:05.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:45:05.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:45:05.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
11:45:05.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:45:05.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:45:05.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:45:05.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 10:52:50 UTC on ccycloud.quasar-peqjre.root.comops.site
11:45:05.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679900
11:45:05.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:45:05.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:05.692844 11125 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:45:05.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:05.693739 11125 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:45:05.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:05.711887 11134 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:45:06.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:05.712316 11132 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:45:06.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:05.715308 11131 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:45:06.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:06.923403 11133 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
11:45:06.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:06.923453 11125 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
11:45:06.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:06.928107 11125 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:45:06.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:06.930877 11125 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:45:06.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:06.932334 11125 hybrid_clock.cc:648] HybridClock initialized: now 1790682306932282 us; error 57 us; skew 500 ppm
11:45:06.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:06.933213 11125 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:45:06.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:45:06 dist-test-slave-8d15 krb5kdc[11110](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682306, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.214.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:45:06.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:06.939033 11125 init.cc:377] Logged in from keytab as kudu/127.10.214.190@KRBTEST.COM (short username kudu)
11:45:06.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:06.944445 11125 webserver.cc:460] Webserver started at http://127.10.214.190:35351/ using document root <none> and password file <none>
11:45:06.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:06.945341 11125 fs_manager.cc:359] Metadata directory not provided
11:45:06.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:06.945564 11125 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:45:06.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:06.946058 11125 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
11:45:06.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:06.950659 11125 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-0/data/instance:
11:45:06.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e35b4b67316c413cbdebfb6f5b3c18f3"
11:45:06.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:45:06 on dist-test-slave-8d15"
11:45:06.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:06.951884 11125 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-0/wal/instance:
11:45:06.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e35b4b67316c413cbdebfb6f5b3c18f3"
11:45:06.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:45:06 on dist-test-slave-8d15"
11:45:06.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:06.959244 11125 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.006s sys 0.000s
11:45:06.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:06.964808 11143 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:45:06.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:06.965718 11125 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
11:45:06.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:06.966058 11125 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-0/data,/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-0/wal
11:45:06.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e35b4b67316c413cbdebfb6f5b3c18f3"
11:45:06.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:45:06 on dist-test-slave-8d15"
11:45:06.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:06.966388 11125 fs_report.cc:383] FS layout report
11:45:06.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:45:06.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-0/wal
11:45:06.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-0/wal
11:45:06.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-0/data/data
11:45:06.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:45:06.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:45:06.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:45:06.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:45:06.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:45:06.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:45:06.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:45:06.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:45:06.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:45:06.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:45:06.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:45:07.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:07.025866 11125 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:45:07.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:07.031514 11125 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:45:07.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:07.032092 11125 kserver.cc:163] Server-wide thread pool size limit: 3276
11:45:07.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:07.108323 11125 rpc_server.cc:307] RPC server started. Bound to: 127.10.214.190:41139
11:45:07.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:07.108464 11194 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.214.190:41139 every 8 connection(s)
11:45:07.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:07.111409 11125 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-0/data/info.pb
11:45:07.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:07.113746 11098 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskst9njx/build/tsan/bin/kudu as pid 11125
11:45:07.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:07.114283 11098 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-0/wal/instance
11:45:07.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:07.117925 11195 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
11:45:07.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.10.214.189@KRBTEST.COM; defaulting to no policy
11:45:07.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:07.131829 11195 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.214.190" port: 41139 } has no permanent_uuid. Determining permanent_uuid...
11:45:07.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
11:45:07.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.10.214.189@KRBTEST.COM" created.
11:45:07.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:45:07 dist-test-slave-8d15 krb5kdc[11110](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682306, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.214.190@KRBTEST.COM for kudu/127.10.214.190@KRBTEST.COM
11:45:07.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
11:45:07.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.10.214.189 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/krb5kdc/kudu_127.10.214.189.keytab.
11:45:07.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.10.214.189 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/krb5kdc/kudu_127.10.214.189.keytab.
11:45:07.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:07.161790 11098 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskst9njx/build/tsan/bin/kudu
11:45:07.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskst9njx/build/tsan/bin/kudu
11:45:07.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-1/wal
11:45:07.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-1/data
11:45:07.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:45:07.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:45:07.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:45:07.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:45:07.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:45:07.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:45:07.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-1/logs
11:45:07.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-1/data/info.pb
11:45:07.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:45:07.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:45:07.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:45:07.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:45:07.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:45:07.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:45:07.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:45:07.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:45:07.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
11:45:07.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:45:07.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:45:07.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:45:07.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.214.189:36301
11:45:07.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.214.189
11:45:07.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:45:07.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/krb5kdc/kudu_127.10.214.189.keytab
11:45:07.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.10.214.189
11:45:07.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:45:07.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
11:45:07.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
11:45:07.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.214.148:40277
11:45:07.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:45:07.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:45:07.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:45:07.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:45:07.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.214.190:41139,127.10.214.189:36301,127.10.214.188:40593
11:45:07.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:45:07.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45657/jwks.json
11:45:07.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
11:45:07.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:07.170375 11195 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.214.189" port: 36301 } has no permanent_uuid. Determining permanent_uuid...
11:45:07.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:07.173790 11146 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.214.189:36301: connect: Connection refused (error 111)
11:45:07.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:07.178208 11195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.214.189:36301: Network error: Client connection negotiation failed: client connection to 127.10.214.189:36301: connect: Connection refused (error 111)
11:45:07.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:07.227707 11195 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.214.189" port: 36301 } attempt: 1
11:45:07.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:07.231280 11195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.214.189:36301: Network error: Client connection negotiation failed: client connection to 127.10.214.189:36301: connect: Connection refused (error 111)
11:45:07.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:07.299724 11195 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.214.189" port: 36301 } attempt: 2
11:45:07.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:07.304507 11195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.214.189:36301: Network error: Client connection negotiation failed: client connection to 127.10.214.189:36301: connect: Connection refused (error 111)
11:45:07.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:07.395983 11195 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.214.189" port: 36301 } attempt: 3
11:45:07.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:07.399922 11195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.214.189:36301: Network error: Client connection negotiation failed: client connection to 127.10.214.189:36301: connect: Connection refused (error 111)
11:45:07.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:07.497650 11201 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:45:07.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:07.498160 11201 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:45:07.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:07.498701 11201 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:45:07.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:07.528254 11201 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
11:45:07.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:07.528465 11201 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
11:45:07.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:07.528692 11201 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:45:07.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:07.528861 11201 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
11:45:07.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:07.529028 11201 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
11:45:07.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:07.529224 11201 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
11:45:07.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:07.529363 11201 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45657/jwks.json
11:45:07.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:07.543318 11195 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.214.189" port: 36301 } attempt: 4
11:45:07.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:07.546819 11195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.214.189:36301: Network error: Client connection negotiation failed: client connection to 127.10.214.189:36301: connect: Connection refused (error 111)
11:45:07.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:07.564756 11201 master_runner.cc:386] Master server non-default flags:
11:45:07.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:45:07.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.214.148:40277
11:45:07.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:45:07.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:45:07.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-1/data
11:45:07.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-1/wal
11:45:07.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
11:45:07.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:45:07.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.214.190:41139,127.10.214.189:36301,127.10.214.188:40593
11:45:07.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/krb5kdc/kudu_127.10.214.189.keytab
11:45:07.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.10.214.189
11:45:07.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:45:07.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:45:07.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:45:07.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.214.189:36301
11:45:07.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:45:07.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:45:07.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:45:07.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45657/jwks.json
11:45:07.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:45:07.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
11:45:07.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
11:45:07.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:45:07.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:45:07.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-1/data/info.pb
11:45:07.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.214.189
11:45:07.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:45:07.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:45:07.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:45:07.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:45:07.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:45:07.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:45:07.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-1/logs
11:45:07.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:45:07.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:45:07.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:45:07.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
11:45:07.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:45:07.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:45:07.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:45:07.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 10:52:50 UTC on ccycloud.quasar-peqjre.root.comops.site
11:45:07.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679900
11:45:07.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:45:07.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:07.565850 11201 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:45:07.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:07.566767 11201 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:45:07.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:07.584822 11208 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:45:07.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:07.846215 11195 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.214.189" port: 36301 } attempt: 5
11:45:07.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:07.849710 11195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.214.189:36301: Network error: Client connection negotiation failed: client connection to 127.10.214.189:36301: connect: Connection refused (error 111)
11:45:08.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:08.397099 11195 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.214.189" port: 36301 } attempt: 6
11:45:08.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:08.404541 11195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.214.189:36301: Network error: Client connection negotiation failed: client connection to 127.10.214.189:36301: connect: Connection refused (error 111)
11:45:08.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:07.587921 11209 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:45:08.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:07.589612 11201 server_base.cc:1034] running on GCE node
11:45:08.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:07.584860 11211 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:45:08.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:08.842726 11201 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:45:08.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:08.845779 11201 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:45:08.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:08.847186 11201 hybrid_clock.cc:648] HybridClock initialized: now 1790682308847137 us; error 66 us; skew 500 ppm
11:45:08.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:08.848024 11201 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:45:08.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:45:08 dist-test-slave-8d15 krb5kdc[11110](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682308, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.214.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:45:08.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:08.854079 11201 init.cc:377] Logged in from keytab as kudu/127.10.214.189@KRBTEST.COM (short username kudu)
11:45:08.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:08.859278 11201 webserver.cc:460] Webserver started at http://127.10.214.189:33505/ using document root <none> and password file <none>
11:45:08.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:08.860270 11201 fs_manager.cc:359] Metadata directory not provided
11:45:08.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:08.860492 11201 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:45:08.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:08.861002 11201 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
11:45:08.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:08.866036 11201 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-1/data/instance:
11:45:08.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aa381d346e1c45879e9319334ad9b2e4"
11:45:08.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:45:08 on dist-test-slave-8d15"
11:45:08.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:08.867206 11201 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-1/wal/instance:
11:45:08.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aa381d346e1c45879e9319334ad9b2e4"
11:45:08.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:45:08 on dist-test-slave-8d15"
11:45:08.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:08.874854 11201 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
11:45:08.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:08.880414 11221 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:45:08.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:08.881523 11201 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
11:45:08.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:08.881862 11201 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-1/data,/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-1/wal
11:45:08.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aa381d346e1c45879e9319334ad9b2e4"
11:45:08.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:45:08 on dist-test-slave-8d15"
11:45:08.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:08.882213 11201 fs_report.cc:383] FS layout report
11:45:08.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:45:08.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-1/wal
11:45:08.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-1/wal
11:45:08.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-1/data/data
11:45:08.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:45:08.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:45:08.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:45:08.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:45:08.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:45:08.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:45:08.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:45:08.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:45:08.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:45:08.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:45:08.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:45:08.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:08.925761 11201 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:45:08.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:08.930862 11201 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:45:08.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:08.931376 11201 kserver.cc:163] Server-wide thread pool size limit: 3276
11:45:09.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:09.010676 11201 rpc_server.cc:307] RPC server started. Bound to: 127.10.214.189:36301
11:45:09.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:09.010726 11272 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.214.189:36301 every 8 connection(s)
11:45:09.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:09.014518 11201 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-1/data/info.pb
11:45:09.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:09.016776 11098 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskst9njx/build/tsan/bin/kudu as pid 11201
11:45:09.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:09.017270 11098 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-1/wal/instance
11:45:09.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:09.021727 11273 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
11:45:09.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.10.214.188@KRBTEST.COM; defaulting to no policy
11:45:09.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
11:45:09.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.10.214.188@KRBTEST.COM" created.
11:45:09.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:09.038383 11273 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.214.190" port: 41139 } has no permanent_uuid. Determining permanent_uuid...
11:45:09.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
11:45:09.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.10.214.188 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/krb5kdc/kudu_127.10.214.188.keytab.
11:45:09.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.10.214.188 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/krb5kdc/kudu_127.10.214.188.keytab.
11:45:09.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:09.063661 11098 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskst9njx/build/tsan/bin/kudu
11:45:09.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskst9njx/build/tsan/bin/kudu
11:45:09.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-2/wal
11:45:09.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-2/data
11:45:09.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:45:09.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:45:09.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:45:09.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:45:09.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:45:09.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:45:09.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-2/logs
11:45:09.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-2/data/info.pb
11:45:09.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:45:09.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:45:09.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:45:09.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:45:09.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:45:09.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:45:09.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:45:09.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:45:09.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
11:45:09.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:45:09.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:45:09.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:45:09.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.214.188:40593
11:45:09.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.214.188
11:45:09.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:45:09.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/krb5kdc/kudu_127.10.214.188.keytab
11:45:09.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.10.214.188
11:45:09.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:45:09.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
11:45:09.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
11:45:09.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.214.148:40277
11:45:09.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:45:09.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:45:09.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:45:09.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:45:09.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.214.190:41139,127.10.214.189:36301,127.10.214.188:40593
11:45:09.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:45:09.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45657/jwks.json
11:45:09.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
11:45:09.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:45:09 dist-test-slave-8d15 krb5kdc[11110](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682308, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.214.189@KRBTEST.COM for kudu/127.10.214.190@KRBTEST.COM
11:45:09.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:09.081642 11273 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.214.189" port: 36301 } has no permanent_uuid. Determining permanent_uuid...
11:45:09.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:45:09 dist-test-slave-8d15 krb5kdc[11110](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682308, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.214.189@KRBTEST.COM for kudu/127.10.214.189@KRBTEST.COM
11:45:09.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:09.120276 11273 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.214.188" port: 40593 } has no permanent_uuid. Determining permanent_uuid...
11:45:09.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:09.123098 11222 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.214.188:40593: connect: Connection refused (error 111)
11:45:09.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:09.127857 11273 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.214.188:40593: Network error: Client connection negotiation failed: client connection to 127.10.214.188:40593: connect: Connection refused (error 111)
11:45:09.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:09.177347 11273 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.214.188" port: 40593 } attempt: 1
11:45:09.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:09.181344 11273 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.214.188:40593: Network error: Client connection negotiation failed: client connection to 127.10.214.188:40593: connect: Connection refused (error 111)
11:45:09.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:09.249756 11273 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.214.188" port: 40593 } attempt: 2
11:45:09.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:09.253690 11273 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.214.188:40593: Network error: Client connection negotiation failed: client connection to 127.10.214.188:40593: connect: Connection refused (error 111)
11:45:09.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:09.345073 11273 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.214.188" port: 40593 } attempt: 3
11:45:09.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:09.349368 11273 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.214.188:40593: Network error: Client connection negotiation failed: client connection to 127.10.214.188:40593: connect: Connection refused (error 111)
11:45:09.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:09.385705 11279 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:45:09.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:09.386308 11279 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:45:09.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:09.386970 11279 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:45:09.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:09.419129 11279 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
11:45:09.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:09.419723 11279 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
11:45:09.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:09.420348 11279 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:45:09.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:09.420856 11279 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
11:45:09.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:09.421172 11279 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
11:45:09.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:09.421478 11279 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
11:45:09.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:09.421720 11279 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45657/jwks.json
11:45:09.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:09.463382 11279 master_runner.cc:386] Master server non-default flags:
11:45:09.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:45:09.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.214.148:40277
11:45:09.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:45:09.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:45:09.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-2/data
11:45:09.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-2/wal
11:45:09.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
11:45:09.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:45:09.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.214.190:41139,127.10.214.189:36301,127.10.214.188:40593
11:45:09.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/krb5kdc/kudu_127.10.214.188.keytab
11:45:09.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.10.214.188
11:45:09.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:45:09.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:45:09.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:45:09.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.214.188:40593
11:45:09.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:45:09.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:45:09.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:45:09.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45657/jwks.json
11:45:09.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:45:09.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
11:45:09.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
11:45:09.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:45:09.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:45:09.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-2/data/info.pb
11:45:09.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.214.188
11:45:09.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:45:09.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:45:09.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:45:09.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:45:09.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:45:09.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:45:09.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-2/logs
11:45:09.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:45:09.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:45:09.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:45:09.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
11:45:09.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:45:09.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:45:09.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:45:09.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 10:52:50 UTC on ccycloud.quasar-peqjre.root.comops.site
11:45:09.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679900
11:45:09.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:45:09.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:09.464671 11279 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:45:09.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:09.464932 11195 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.214.189" port: 36301 } attempt: 7
11:45:09.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:09.465682 11279 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:45:09.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:45:09 dist-test-slave-8d15 krb5kdc[11110](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682306, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.214.190@KRBTEST.COM for kudu/127.10.214.189@KRBTEST.COM
11:45:09.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:09.488461 11289 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:45:09.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:09.492818 11273 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.214.188" port: 40593 } attempt: 4
11:45:09.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:09.497512 11273 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.214.188:40593: Network error: Client connection negotiation failed: client connection to 127.10.214.188:40593: connect: Connection refused (error 111)
11:45:09.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:09.497992 11195 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.214.188" port: 40593 } has no permanent_uuid. Determining permanent_uuid...
11:45:09.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:09.503638 11195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.214.188:40593: Network error: Client connection negotiation failed: client connection to 127.10.214.188:40593: connect: Connection refused (error 111)
11:45:09.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:09.562084 11195 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.214.188" port: 40593 } attempt: 1
11:45:09.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:09.566084 11195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.214.188:40593: Network error: Client connection negotiation failed: client connection to 127.10.214.188:40593: connect: Connection refused (error 111)
11:45:09.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:09.647454 11195 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.214.188" port: 40593 } attempt: 2
11:45:09.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:09.651852 11195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.214.188:40593: Network error: Client connection negotiation failed: client connection to 127.10.214.188:40593: connect: Connection refused (error 111)
11:45:09.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:09.737257 11195 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.214.188" port: 40593 } attempt: 3
11:45:09.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:09.741847 11195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.214.188:40593: Network error: Client connection negotiation failed: client connection to 127.10.214.188:40593: connect: Connection refused (error 111)
11:45:09.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:09.796967 11273 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.214.188" port: 40593 } attempt: 5
11:45:09.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:09.801492 11273 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.214.188:40593: Network error: Client connection negotiation failed: client connection to 127.10.214.188:40593: connect: Connection refused (error 111)
11:45:09.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:09.882310 11195 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.214.188" port: 40593 } attempt: 4
11:45:09.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:09.887573 11195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.214.188:40593: Network error: Client connection negotiation failed: client connection to 127.10.214.188:40593: connect: Connection refused (error 111)
11:45:10.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:10.171020 11195 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.214.188" port: 40593 } attempt: 5
11:45:10.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:10.174911 11195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.214.188:40593: Network error: Client connection negotiation failed: client connection to 127.10.214.188:40593: connect: Connection refused (error 111)
11:45:10.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:10.348896 11273 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.214.188" port: 40593 } attempt: 6
11:45:10.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:10.355386 11273 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.214.188:40593: Network error: Client connection negotiation failed: client connection to 127.10.214.188:40593: connect: Connection refused (error 111)
11:45:10.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:10.727452 11195 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.214.188" port: 40593 } attempt: 6
11:45:10.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:10.734965 11195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.214.188:40593: Network error: Client connection negotiation failed: client connection to 127.10.214.188:40593: connect: Connection refused (error 111)
11:45:10.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:09.489890 11291 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:45:10.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:09.490510 11279 server_base.cc:1034] running on GCE node
11:45:10.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:09.490727 11288 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:45:10.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:10.772235 11279 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:45:10.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:10.774971 11279 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:45:10.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:10.776337 11279 hybrid_clock.cc:648] HybridClock initialized: now 1790682310776307 us; error 46 us; skew 500 ppm
11:45:10.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:10.777197 11279 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:45:10.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:45:10 dist-test-slave-8d15 krb5kdc[11110](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682310, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.214.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:45:10.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:10.784009 11279 init.cc:377] Logged in from keytab as kudu/127.10.214.188@KRBTEST.COM (short username kudu)
11:45:10.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:10.789883 11279 webserver.cc:460] Webserver started at http://127.10.214.188:38153/ using document root <none> and password file <none>
11:45:10.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:10.791167 11279 fs_manager.cc:359] Metadata directory not provided
11:45:10.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:10.791589 11279 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:45:10.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:10.792171 11279 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
11:45:10.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:10.799130 11279 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-2/data/instance:
11:45:10.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "db14f371e39f4c28a890e1b83b8634ce"
11:45:10.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:45:10 on dist-test-slave-8d15"
11:45:10.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:10.800591 11279 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-2/wal/instance:
11:45:10.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "db14f371e39f4c28a890e1b83b8634ce"
11:45:10.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:45:10 on dist-test-slave-8d15"
11:45:10.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:10.808851 11279 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.004s sys 0.004s
11:45:10.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:10.815205 11306 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:45:10.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:10.816529 11279 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
11:45:10.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:10.816804 11279 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-2/data,/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-2/wal
11:45:10.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "db14f371e39f4c28a890e1b83b8634ce"
11:45:10.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:45:10 on dist-test-slave-8d15"
11:45:10.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:10.817124 11279 fs_report.cc:383] FS layout report
11:45:10.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:45:10.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-2/wal
11:45:10.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-2/wal
11:45:10.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-2/data/data
11:45:10.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:45:10.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:45:10.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:45:10.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:45:10.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:45:10.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:45:10.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:45:10.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:45:10.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:45:10.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:45:10.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:45:10.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:10.860006 11279 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:45:10.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:10.866763 11279 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:45:10.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:10.867206 11279 kserver.cc:163] Server-wide thread pool size limit: 3276
11:45:10.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:10.948784 11279 rpc_server.cc:307] RPC server started. Bound to: 127.10.214.188:40593
11:45:10.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:10.948918 11358 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.214.188:40593 every 8 connection(s)
11:45:10.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:10.951901 11279 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-2/data/info.pb
11:45:10.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:10.957804 11359 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
11:45:10.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:10.958190 11098 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskst9njx/build/tsan/bin/kudu as pid 11279
11:45:10.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:10.958525 11098 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-2/wal/instance
11:45:10.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:10.974515 11359 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.214.190" port: 41139 } has no permanent_uuid. Determining permanent_uuid...
11:45:11.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:45:10 dist-test-slave-8d15 krb5kdc[11110](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682310, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.214.188@KRBTEST.COM for kudu/127.10.214.190@KRBTEST.COM
11:45:11.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:45:10 dist-test-slave-8d15 krb5kdc[11110](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682305, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.10.214.188@KRBTEST.COM
11:45:11.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:11.021049 11359 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.214.189" port: 36301 } has no permanent_uuid. Determining permanent_uuid...
11:45:11.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:45:11 dist-test-slave-8d15 krb5kdc[11110](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682310, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.214.188@KRBTEST.COM for kudu/127.10.214.189@KRBTEST.COM
11:45:11.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:45:11 dist-test-slave-8d15 krb5kdc[11110](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682305, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.10.214.189@KRBTEST.COM
11:45:11.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:11.058638 11359 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.214.188" port: 40593 } has no permanent_uuid. Determining permanent_uuid...
11:45:11.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:45:11 dist-test-slave-8d15 krb5kdc[11110](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682305, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.10.214.190@KRBTEST.COM
11:45:11.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:45:11 dist-test-slave-8d15 krb5kdc[11110](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682310, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.214.188@KRBTEST.COM for kudu/127.10.214.188@KRBTEST.COM
11:45:11.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:11.090256 11098 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
11:45:11.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:11.091717 11359 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce: Bootstrap starting.
11:45:11.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:11.097045 11359 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce: Neither blocks nor log segments found. Creating new log.
11:45:11.098 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:45:11.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:11.098524 11359 log.cc:824] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce: Log is configured to *not* fsync() on all Append() calls
11:45:11.099 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
11:45:11.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:11.103947 11359 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce: No bootstrap required, opened a new log
11:45:11.112 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
11:45:11.113 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
11:45:11.118 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.10.214.190"
port: 41139
}
bound_http_address {
host: "127.10.214.190"
port: 35351
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.10.214.189"
port: 36301
}
bound_http_address {
host: "127.10.214.189"
port: 33505
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.10.214.188"
port: 40593
}
bound_http_address {
host: "127.10.214.188"
port: 38153
}
}
}
11:45:11.119 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
11:45:11.120 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:45:11.120 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
11:45:11.121 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
11:45:11.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:11.133816 11359 raft_consensus.cc:348] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e35b4b67316c413cbdebfb6f5b3c18f3" member_type: VOTER last_known_addr { host: "127.10.214.190" port: 41139 } } peers { permanent_uuid: "aa381d346e1c45879e9319334ad9b2e4" member_type: VOTER last_known_addr { host: "127.10.214.189" port: 36301 } } peers { permanent_uuid: "db14f371e39f4c28a890e1b83b8634ce" member_type: VOTER last_known_addr { host: "127.10.214.188" port: 40593 } }
11:45:11.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:11.134438 11359 raft_consensus.cc:374] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
11:45:11.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:11.134724 11359 raft_consensus.cc:729] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: db14f371e39f4c28a890e1b83b8634ce, State: Initialized, Role: FOLLOWER
11:45:11.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:11.135617 11359 consensus_queue.cc:260] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e35b4b67316c413cbdebfb6f5b3c18f3" member_type: VOTER last_known_addr { host: "127.10.214.190" port: 41139 } } peers { permanent_uuid: "aa381d346e1c45879e9319334ad9b2e4" member_type: VOTER last_known_addr { host: "127.10.214.189" port: 36301 } } peers { permanent_uuid: "db14f371e39f4c28a890e1b83b8634ce" member_type: VOTER last_known_addr { host: "127.10.214.188" port: 40593 } }
11:45:11.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:11.138870 11369 sys_catalog.cc:455] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e35b4b67316c413cbdebfb6f5b3c18f3" member_type: VOTER last_known_addr { host: "127.10.214.190" port: 41139 } } peers { permanent_uuid: "aa381d346e1c45879e9319334ad9b2e4" member_type: VOTER last_known_addr { host: "127.10.214.189" port: 36301 } } peers { permanent_uuid: "db14f371e39f4c28a890e1b83b8634ce" member_type: VOTER last_known_addr { host: "127.10.214.188" port: 40593 } } }
11:45:11.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:11.139905 11369 sys_catalog.cc:458] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce [sys.catalog]: This master's current role is: FOLLOWER
11:45:11.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:11.140691 11359 sys_catalog.cc:564] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce [sys.catalog]: configured and running, proceeding with master startup.
11:45:11.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:45:11 dist-test-slave-8d15 krb5kdc[11110](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682311, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:45:11.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
11:45:11.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:11.170670 11381 catalog_manager.cc:1558] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:45:11.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:11.170980 11381 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:45:11.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:11.166810 11279 master_runner.cc:186] Error getting master registration for 127.10.214.190:41139: OK, instance_id { permanent_uuid: "e35b4b67316c413cbdebfb6f5b3c18f3" instance_seqno: 1790682307087051 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
11:45:11.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:11.192304 11279 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
11:45:11.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:11.252583 11098 mini_kdc.cc:317] Time spent kinit for test-admin: real 0.130s user 0.006s sys 0.007s
11:45:11.253 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:45:11.255 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/krb5kdc/krb5cc
11:45:11.263 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
11:45:11.264 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/krb5kdc/krb5cc
11:45:11.265 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
11:45:11.269 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
11:45:11.271 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
11:45:11.271 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
11:45:11.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:45:11 dist-test-slave-8d15 krb5kdc[11110](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682311, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:45:11.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
11:45:11.354 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:45:11.354 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/krb5kdc/krb5cc
11:45:11.356 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
11:45:11.357 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.10.214.189:36301
11:45:11.359 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
11:45:11.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:11.359813 11098 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskst9njx/build/tsan/bin/kudu with pid 11201
11:45:11.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:11.373562 11313 tablet.cc:2369] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce: Can't schedule compaction. Clean time has not been advanced past its initial value.
11:45:11.392 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:45:11.392 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.10.214.190:41139
11:45:11.392 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
11:45:11.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:11.393062 11098 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskst9njx/build/tsan/bin/kudu with pid 11125
11:45:11.426 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:45:11.426 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.10.214.188:40593
11:45:11.426 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
11:45:11.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:11.427114 11098 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskst9njx/build/tsan/bin/kudu with pid 11279
11:45:11.450 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:45:11.450 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.10.214.189:36301
11:45:11.451 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
11:45:11.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:11.453153 11098 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskst9njx/build/tsan/bin/kudu
11:45:11.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskst9njx/build/tsan/bin/kudu
11:45:11.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-1/wal
11:45:11.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-1/data
11:45:11.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:45:11.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:45:11.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:45:11.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:45:11.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:45:11.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:45:11.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-1/logs
11:45:11.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-1/data/info.pb
11:45:11.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:45:11.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:45:11.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:45:11.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:45:11.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:45:11.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:45:11.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:45:11.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:45:11.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
11:45:11.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:45:11.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:45:11.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:45:11.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.214.189:36301
11:45:11.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.214.189
11:45:11.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33505
11:45:11.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/krb5kdc/kudu_127.10.214.189.keytab
11:45:11.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.10.214.189
11:45:11.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:45:11.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
11:45:11.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
11:45:11.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.214.148:40277
11:45:11.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:45:11.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:45:11.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:45:11.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:45:11.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.214.190:41139,127.10.214.189:36301,127.10.214.188:40593
11:45:11.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:45:11.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45657/jwks.json
11:45:11.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
11:45:11.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:11.779805 11385 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:45:11.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:11.780572 11385 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:45:11.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:11.781545 11385 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:45:11.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:11.822899 11385 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
11:45:11.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:11.823179 11385 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
11:45:11.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:11.823614 11385 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:45:11.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:11.823944 11385 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
11:45:11.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:11.824254 11385 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
11:45:11.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:11.824548 11385 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
11:45:11.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:11.824769 11385 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45657/jwks.json
11:45:11.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:11.863952 11385 master_runner.cc:386] Master server non-default flags:
11:45:11.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:45:11.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.214.148:40277
11:45:11.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:45:11.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:45:11.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-1/data
11:45:11.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-1/wal
11:45:11.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
11:45:11.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:45:11.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.214.190:41139,127.10.214.189:36301,127.10.214.188:40593
11:45:11.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/krb5kdc/kudu_127.10.214.189.keytab
11:45:11.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.10.214.189
11:45:11.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:45:11.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:45:11.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:45:11.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.214.189:36301
11:45:11.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:45:11.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:45:11.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:45:11.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45657/jwks.json
11:45:11.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:45:11.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
11:45:11.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
11:45:11.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:45:11.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:45:11.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-1/data/info.pb
11:45:11.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.214.189
11:45:11.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33505
11:45:11.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:45:11.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:45:11.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:45:11.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:45:11.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:45:11.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-1/logs
11:45:11.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:45:11.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:45:11.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:45:11.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
11:45:11.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:45:11.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:45:11.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:45:11.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 10:52:50 UTC on ccycloud.quasar-peqjre.root.comops.site
11:45:11.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679900
11:45:11.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:45:11.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:11.865435 11385 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:45:11.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:11.866590 11385 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:45:11.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:11.886502 11393 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:45:13.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:11.886565 11392 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:45:13.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:11.887715 11395 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:45:13.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:11.888706 11385 server_base.cc:1034] running on GCE node
11:45:13.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:13.130971 11385 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:45:13.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:13.133490 11385 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:45:13.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:13.134878 11385 hybrid_clock.cc:648] HybridClock initialized: now 1790682313134842 us; error 44 us; skew 500 ppm
11:45:13.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:13.135777 11385 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:45:13.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:45:13 dist-test-slave-8d15 krb5kdc[11110](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682313, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.214.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:45:13.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:13.141258 11385 init.cc:377] Logged in from keytab as kudu/127.10.214.189@KRBTEST.COM (short username kudu)
11:45:13.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:13.146862 11385 webserver.cc:460] Webserver started at http://127.10.214.189:33505/ using document root <none> and password file <none>
11:45:13.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:13.147780 11385 fs_manager.cc:359] Metadata directory not provided
11:45:13.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:13.147969 11385 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:45:13.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:13.155860 11385 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.007s sys 0.001s
11:45:13.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:13.159827 11404 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:45:13.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:13.160818 11385 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
11:45:13.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:13.161089 11385 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-1/data,/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-1/wal
11:45:13.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aa381d346e1c45879e9319334ad9b2e4"
11:45:13.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:45:08 on dist-test-slave-8d15"
11:45:13.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:13.163229 11385 fs_report.cc:383] FS layout report
11:45:13.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:45:13.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-1/wal
11:45:13.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-1/wal
11:45:13.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-1/data/data
11:45:13.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:45:13.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:45:13.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:45:13.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:45:13.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:45:13.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:45:13.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:45:13.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:45:13.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:45:13.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:45:13.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:45:13.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:13.200497 11385 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:45:13.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:13.206717 11385 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:45:13.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:13.207111 11385 kserver.cc:163] Server-wide thread pool size limit: 3276
11:45:13.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:13.280635 11385 rpc_server.cc:307] RPC server started. Bound to: 127.10.214.189:36301
11:45:13.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:13.280791 11455 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.214.189:36301 every 8 connection(s)
11:45:13.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:13.284273 11385 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-1/data/info.pb
11:45:13.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:13.287966 11098 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskst9njx/build/tsan/bin/kudu as pid 11385
11:45:13.288 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:45:13.289 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.10.214.190:41139
11:45:13.289 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
11:45:13.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:13.290705 11098 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskst9njx/build/tsan/bin/kudu
11:45:13.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskst9njx/build/tsan/bin/kudu
11:45:13.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-0/wal
11:45:13.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-0/data
11:45:13.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:45:13.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:45:13.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:45:13.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:45:13.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:45:13.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:45:13.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-0/logs
11:45:13.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-0/data/info.pb
11:45:13.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:45:13.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:45:13.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:45:13.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:45:13.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:45:13.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:45:13.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:45:13.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:45:13.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
11:45:13.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:45:13.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:45:13.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:45:13.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.214.190:41139
11:45:13.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.214.190
11:45:13.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35351
11:45:13.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/krb5kdc/kudu_127.10.214.190.keytab
11:45:13.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.10.214.190
11:45:13.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:45:13.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
11:45:13.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
11:45:13.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.214.148:40277
11:45:13.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:45:13.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:45:13.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:45:13.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:45:13.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.214.190:41139,127.10.214.189:36301,127.10.214.188:40593
11:45:13.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:45:13.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45657/jwks.json
11:45:13.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
11:45:13.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:13.298441 11456 sys_catalog.cc:263] Verifying existing consensus state
11:45:13.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260929 11:45:13.299209 11456 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
11:45:13.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:13.299657 11385 master.cc:537] Master@127.10.214.189:36301 shutting down...
11:45:13.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:13.366226 11385 master.cc:552] Master@127.10.214.189:36301 shutdown complete.
11:45:13.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
11:45:13.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:13.621210 11458 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:45:13.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:13.621733 11458 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:45:13.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:13.622299 11458 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:45:13.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:13.655144 11458 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
11:45:13.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:13.655345 11458 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
11:45:13.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:13.655678 11458 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:45:13.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:13.656023 11458 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
11:45:13.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:13.656332 11458 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
11:45:13.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:13.656639 11458 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
11:45:13.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:13.656867 11458 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45657/jwks.json
11:45:13.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:13.696602 11458 master_runner.cc:386] Master server non-default flags:
11:45:13.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:45:13.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.214.148:40277
11:45:13.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:45:13.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:45:13.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-0/data
11:45:13.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-0/wal
11:45:13.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
11:45:13.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:45:13.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.214.190:41139,127.10.214.189:36301,127.10.214.188:40593
11:45:13.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/krb5kdc/kudu_127.10.214.190.keytab
11:45:13.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.10.214.190
11:45:13.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:45:13.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:45:13.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:45:13.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.214.190:41139
11:45:13.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:45:13.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:45:13.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:45:13.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45657/jwks.json
11:45:13.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:45:13.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
11:45:13.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
11:45:13.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:45:13.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:45:13.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-0/data/info.pb
11:45:13.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.214.190
11:45:13.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35351
11:45:13.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:45:13.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:45:13.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:45:13.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:45:13.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:45:13.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-0/logs
11:45:13.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:45:13.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:45:13.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:45:13.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
11:45:13.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:45:13.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:45:13.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:45:13.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 10:52:50 UTC on ccycloud.quasar-peqjre.root.comops.site
11:45:13.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679900
11:45:13.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:45:13.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:13.697696 11458 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:45:13.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:13.698598 11458 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:45:13.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:13.718140 11464 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:45:14.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:13.718140 11465 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:45:14.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:13.718155 11467 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:45:14.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:13.720538 11458 server_base.cc:1034] running on GCE node
11:45:14.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:14.942657 11458 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:45:14.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:14.945173 11458 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:45:14.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:14.946509 11458 hybrid_clock.cc:648] HybridClock initialized: now 1790682314946470 us; error 57 us; skew 500 ppm
11:45:14.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:14.947249 11458 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:45:14.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:45:14 dist-test-slave-8d15 krb5kdc[11110](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682314, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.214.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:45:14.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:14.958055 11458 init.cc:377] Logged in from keytab as kudu/127.10.214.190@KRBTEST.COM (short username kudu)
11:45:14.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:14.963150 11458 webserver.cc:460] Webserver started at http://127.10.214.190:35351/ using document root <none> and password file <none>
11:45:14.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:14.964047 11458 fs_manager.cc:359] Metadata directory not provided
11:45:14.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:14.964228 11458 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:45:14.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:14.971989 11458 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.004s sys 0.003s
11:45:14.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:14.976266 11478 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:45:14.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:14.977319 11458 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
11:45:14.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:14.977569 11458 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-0/data,/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-0/wal
11:45:14.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e35b4b67316c413cbdebfb6f5b3c18f3"
11:45:14.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:45:06 on dist-test-slave-8d15"
11:45:14.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:14.979853 11458 fs_report.cc:383] FS layout report
11:45:14.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:45:14.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-0/wal
11:45:14.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-0/wal
11:45:14.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-0/data/data
11:45:14.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:45:14.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:45:14.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:45:14.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:45:14.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:45:14.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:45:14.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:45:14.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:45:14.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:45:14.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:45:14.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:45:15.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:15.022293 11458 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:45:15.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:15.027738 11458 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:45:15.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:15.028139 11458 kserver.cc:163] Server-wide thread pool size limit: 3276
11:45:15.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:15.101840 11458 rpc_server.cc:307] RPC server started. Bound to: 127.10.214.190:41139
11:45:15.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:15.101927 11529 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.214.190:41139 every 8 connection(s)
11:45:15.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:15.104483 11458 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-0/data/info.pb
11:45:15.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:15.106238 11098 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskst9njx/build/tsan/bin/kudu as pid 11458
11:45:15.107 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:45:15.107 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.10.214.188:40593
11:45:15.108 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
11:45:15.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:15.109623 11098 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskst9njx/build/tsan/bin/kudu
11:45:15.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskst9njx/build/tsan/bin/kudu
11:45:15.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-2/wal
11:45:15.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-2/data
11:45:15.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:45:15.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:45:15.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:45:15.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:45:15.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:45:15.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:45:15.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-2/logs
11:45:15.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-2/data/info.pb
11:45:15.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:45:15.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:45:15.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:45:15.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:45:15.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:45:15.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:45:15.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:45:15.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:45:15.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
11:45:15.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:45:15.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:45:15.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:45:15.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.214.188:40593
11:45:15.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.214.188
11:45:15.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38153
11:45:15.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/krb5kdc/kudu_127.10.214.188.keytab
11:45:15.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.10.214.188
11:45:15.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:45:15.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
11:45:15.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
11:45:15.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.214.148:40277
11:45:15.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:45:15.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:45:15.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:45:15.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:45:15.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.214.190:41139,127.10.214.189:36301,127.10.214.188:40593
11:45:15.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:45:15.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45657/jwks.json
11:45:15.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
11:45:15.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:15.115811 11530 sys_catalog.cc:263] Verifying existing consensus state
11:45:15.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260929 11:45:15.116691 11530 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
11:45:15.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:15.117178 11458 master.cc:537] Master@127.10.214.190:41139 shutting down...
11:45:15.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:15.172174 11458 master.cc:552] Master@127.10.214.190:41139 shutdown complete.
11:45:15.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
11:45:15.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:15.426180 11532 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:45:15.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:15.426672 11532 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:45:15.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:15.427237 11532 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:45:15.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:15.459968 11532 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
11:45:15.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:15.460166 11532 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
11:45:15.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:15.460414 11532 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:45:15.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:15.460609 11532 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
11:45:15.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:15.460791 11532 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
11:45:15.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:15.460980 11532 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
11:45:15.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:15.461126 11532 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45657/jwks.json
11:45:15.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:15.498093 11532 master_runner.cc:386] Master server non-default flags:
11:45:15.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:45:15.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.214.148:40277
11:45:15.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:45:15.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:45:15.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-2/data
11:45:15.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-2/wal
11:45:15.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
11:45:15.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:45:15.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.214.190:41139,127.10.214.189:36301,127.10.214.188:40593
11:45:15.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/krb5kdc/kudu_127.10.214.188.keytab
11:45:15.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.10.214.188
11:45:15.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:45:15.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:45:15.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:45:15.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.214.188:40593
11:45:15.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:45:15.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:45:15.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:45:15.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45657/jwks.json
11:45:15.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:45:15.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
11:45:15.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
11:45:15.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:45:15.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:45:15.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-2/data/info.pb
11:45:15.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.214.188
11:45:15.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38153
11:45:15.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:45:15.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:45:15.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:45:15.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:45:15.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:45:15.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-2/logs
11:45:15.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:45:15.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:45:15.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:45:15.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
11:45:15.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:45:15.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:45:15.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:45:15.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 10:52:50 UTC on ccycloud.quasar-peqjre.root.comops.site
11:45:15.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679900
11:45:15.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:45:15.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:15.499135 11532 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:45:15.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:15.500229 11532 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:45:15.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:15.517903 11541 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:45:16.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:15.518028 11539 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:45:16.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:15.519647 11532 server_base.cc:1034] running on GCE node
11:45:16.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:15.520054 11538 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:45:16.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:16.741777 11532 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:45:16.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:16.744279 11532 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:45:16.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:16.745666 11532 hybrid_clock.cc:648] HybridClock initialized: now 1790682316745634 us; error 38 us; skew 500 ppm
11:45:16.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:16.746570 11532 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:45:16.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:45:16 dist-test-slave-8d15 krb5kdc[11110](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682316, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.214.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:45:16.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:16.752583 11532 init.cc:377] Logged in from keytab as kudu/127.10.214.188@KRBTEST.COM (short username kudu)
11:45:16.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:16.757197 11532 webserver.cc:460] Webserver started at http://127.10.214.188:38153/ using document root <none> and password file <none>
11:45:16.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:16.758025 11532 fs_manager.cc:359] Metadata directory not provided
11:45:16.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:16.758265 11532 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:45:16.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:16.765770 11532 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.005s sys 0.000s
11:45:16.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:16.769995 11549 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:45:16.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:16.771195 11532 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
11:45:16.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:16.771620 11532 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-2/data,/tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-2/wal
11:45:16.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "db14f371e39f4c28a890e1b83b8634ce"
11:45:16.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:45:10 on dist-test-slave-8d15"
11:45:16.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:16.775238 11532 fs_report.cc:383] FS layout report
11:45:16.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:45:16.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-2/wal
11:45:16.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-2/wal
11:45:16.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-2/data/data
11:45:16.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:45:16.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:45:16.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:45:16.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:45:16.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:45:16.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:45:16.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:45:16.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:45:16.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:45:16.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:45:16.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:45:16.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:16.821069 11532 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:45:16.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:16.826457 11532 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:45:16.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:16.826825 11532 kserver.cc:163] Server-wide thread pool size limit: 3276
11:45:16.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:16.896644 11532 rpc_server.cc:307] RPC server started. Bound to: 127.10.214.188:40593
11:45:16.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:16.896764 11601 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.214.188:40593 every 8 connection(s)
11:45:16.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:16.899138 11532 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskst9njx/test-tmp/mini-kudu-cluster9125218997434528026/master-2/data/info.pb
11:45:16.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:16.901216 11098 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskst9njx/build/tsan/bin/kudu as pid 11532
11:45:16.902 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:45:16.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:16.913056 11602 sys_catalog.cc:263] Verifying existing consensus state
11:45:16.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:16.920837 11602 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce: Bootstrap starting.
11:45:16.959 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.10.214.189:36301(127.10.214.189:36301)] connecting to peer
11:45:16.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:16.962497 11602 log.cc:824] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce: Log is configured to *not* fsync() on all Append() calls
11:45:16.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:16.967507 11602 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
11:45:16.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:16.968335 11602 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce: Bootstrap complete.
11:45:17.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:17.004968 11602 raft_consensus.cc:348] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e35b4b67316c413cbdebfb6f5b3c18f3" member_type: VOTER last_known_addr { host: "127.10.214.190" port: 41139 } } peers { permanent_uuid: "aa381d346e1c45879e9319334ad9b2e4" member_type: VOTER last_known_addr { host: "127.10.214.189" port: 36301 } } peers { permanent_uuid: "db14f371e39f4c28a890e1b83b8634ce" member_type: VOTER last_known_addr { host: "127.10.214.188" port: 40593 } }
11:45:17.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:17.008564 11602 raft_consensus.cc:374] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
11:45:17.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:17.008880 11602 raft_consensus.cc:729] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: db14f371e39f4c28a890e1b83b8634ce, State: Initialized, Role: FOLLOWER
11:45:17.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:17.009836 11602 consensus_queue.cc:260] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e35b4b67316c413cbdebfb6f5b3c18f3" member_type: VOTER last_known_addr { host: "127.10.214.190" port: 41139 } } peers { permanent_uuid: "aa381d346e1c45879e9319334ad9b2e4" member_type: VOTER last_known_addr { host: "127.10.214.189" port: 36301 } } peers { permanent_uuid: "db14f371e39f4c28a890e1b83b8634ce" member_type: VOTER last_known_addr { host: "127.10.214.188" port: 40593 } }
11:45:17.021 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.10.214.190:41139(127.10.214.190:41139)] connecting to peer
11:45:17.025 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.10.214.188:40593(127.10.214.188:40593)] connecting to peer
11:45:17.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:17.026984 11602 sys_catalog.cc:564] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce [sys.catalog]: configured and running, proceeding with master startup.
11:45:17.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:17.024632 11606 sys_catalog.cc:455] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e35b4b67316c413cbdebfb6f5b3c18f3" member_type: VOTER last_known_addr { host: "127.10.214.190" port: 41139 } } peers { permanent_uuid: "aa381d346e1c45879e9319334ad9b2e4" member_type: VOTER last_known_addr { host: "127.10.214.189" port: 36301 } } peers { permanent_uuid: "db14f371e39f4c28a890e1b83b8634ce" member_type: VOTER last_known_addr { host: "127.10.214.188" port: 40593 } } }
11:45:17.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:17.036650 11606 sys_catalog.cc:458] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce [sys.catalog]: This master's current role is: FOLLOWER
11:45:17.079 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:17.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:17.079101 11619 catalog_manager.cc:1558] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:45:17.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:17.079435 11619 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:45:17.080 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.214.189:36301(127.10.214.189:36301)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:17.080 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:17.080 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.214.190:41139(127.10.214.190:41139)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:17.081 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.214.190:41139: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:17.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:17.081648 11556 tablet.cc:2369] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce: Can't schedule compaction. Clean time has not been advanced past its initial value.
11:45:17.083 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.214.189:36301: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:17.084 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.10.214.188:40593(127.10.214.188:40593)] Successfully connected to peer
11:45:17.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:17.100296 11532 master_runner.cc:186] Error getting master registration for 127.10.214.190:41139: Network error: Client connection negotiation failed: client connection to 127.10.214.190:41139: connect: Connection refused (error 111),
11:45:17.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:17.131539 11532 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
11:45:17.190 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
11:45:17.200 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.10.214.188
11:45:17.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:17.270006 10449 debug-util.cc:398] Leaking SignalData structure 0x7b08000b5ee0 after lost signal to thread 10386
11:45:17.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:17.270697 10449 debug-util.cc:398] Leaking SignalData structure 0x7b08000bb520 after lost signal to thread 10452
11:45:17.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:45:17 dist-test-slave-8d15 krb5kdc[11110](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682311, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.10.214.188@KRBTEST.COM
11:45:17.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:17.476461 11606 raft_consensus.cc:482] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
11:45:17.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:17.476994 11606 raft_consensus.cc:504] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e35b4b67316c413cbdebfb6f5b3c18f3" member_type: VOTER last_known_addr { host: "127.10.214.190" port: 41139 } } peers { permanent_uuid: "aa381d346e1c45879e9319334ad9b2e4" member_type: VOTER last_known_addr { host: "127.10.214.189" port: 36301 } } peers { permanent_uuid: "db14f371e39f4c28a890e1b83b8634ce" member_type: VOTER last_known_addr { host: "127.10.214.188" port: 40593 } }
11:45:17.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:17.481393 11552 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.214.190:41139: connect: Connection refused (error 111)
11:45:17.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:17.485291 11606 leader_election.cc:288] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e35b4b67316c413cbdebfb6f5b3c18f3 (127.10.214.190:41139), aa381d346e1c45879e9319334ad9b2e4 (127.10.214.189:36301)
11:45:17.488 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x4f5fcdcc, L:/127.0.0.1:45570 - R:/127.10.214.188:40593] using SASL/GSSAPI
11:45:17.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:17.493808 11553 leader_election.cc:334] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer aa381d346e1c45879e9319334ad9b2e4 (127.10.214.189:36301): Network error: Client connection negotiation failed: client connection to 127.10.214.189:36301: connect: Connection refused (error 111)
11:45:17.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:17.494613 11552 leader_election.cc:334] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e35b4b67316c413cbdebfb6f5b3c18f3 (127.10.214.190:41139): Network error: Client connection negotiation failed: client connection to 127.10.214.190:41139: connect: Connection refused (error 111)
11:45:17.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:17.495012 11552 leader_election.cc:302] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: db14f371e39f4c28a890e1b83b8634ce; no voters: aa381d346e1c45879e9319334ad9b2e4, e35b4b67316c413cbdebfb6f5b3c18f3
11:45:17.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:17.495774 11606 raft_consensus.cc:2726] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
11:45:17.506 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593; will retry
11:45:17.528 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.214.189:36301(127.10.214.189:36301)] connecting to peer
11:45:17.530 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.214.190:41139(127.10.214.190:41139)] connecting to peer
11:45:17.537 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:17.537 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.214.189:36301(127.10.214.189:36301)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:17.540 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.214.189:36301: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:17.540 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:17.540 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.214.190:41139(127.10.214.190:41139)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:17.541 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.214.190:41139: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:17.543 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593; will retry
11:45:17.547 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.214.189:36301(127.10.214.189:36301)] connecting to peer
11:45:17.549 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.214.190:41139(127.10.214.190:41139)] connecting to peer
11:45:17.549 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:17.549 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.214.189:36301(127.10.214.189:36301)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:17.550 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.214.189:36301: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:17.552 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:17.552 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.214.190:41139(127.10.214.190:41139)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:17.553 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.214.190:41139: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:17.555 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593; will retry
11:45:17.568 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.214.189:36301(127.10.214.189:36301)] connecting to peer
11:45:17.569 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.214.190:41139(127.10.214.190:41139)] connecting to peer
11:45:17.570 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:17.570 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.214.189:36301(127.10.214.189:36301)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:17.572 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.214.189:36301: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:17.574 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:17.574 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.214.190:41139(127.10.214.190:41139)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:17.575 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.214.190:41139: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:17.577 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593; will retry
11:45:17.608 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.214.189:36301(127.10.214.189:36301)] connecting to peer
11:45:17.609 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.214.190:41139(127.10.214.190:41139)] connecting to peer
11:45:17.610 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:17.611 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.214.189:36301(127.10.214.189:36301)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:17.611 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.214.189:36301: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:17.618 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:17.618 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.214.190:41139(127.10.214.190:41139)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:17.619 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.214.190:41139: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:17.619 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593; will retry
11:45:17.648 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.214.189:36301(127.10.214.189:36301)] connecting to peer
11:45:17.650 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.214.190:41139(127.10.214.190:41139)] connecting to peer
11:45:17.651 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:17.651 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.214.189:36301(127.10.214.189:36301)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:17.653 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.214.189:36301: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:17.655 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:17.655 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.214.190:41139(127.10.214.190:41139)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:17.655 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.214.190:41139: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:17.657 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593; will retry
11:45:17.688 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.214.189:36301(127.10.214.189:36301)] connecting to peer
11:45:17.689 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.214.190:41139(127.10.214.190:41139)] connecting to peer
11:45:17.691 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:17.691 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.214.189:36301(127.10.214.189:36301)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:17.692 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.214.189:36301: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:17.693 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:17.693 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.214.190:41139(127.10.214.190:41139)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:17.694 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.214.190:41139: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:17.696 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593; will retry
11:45:17.808 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.214.189:36301(127.10.214.189:36301)] connecting to peer
11:45:17.809 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.214.190:41139(127.10.214.190:41139)] connecting to peer
11:45:17.810 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:17.811 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.214.189:36301(127.10.214.189:36301)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:17.811 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.214.189:36301: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:17.813 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:17.813 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.214.190:41139(127.10.214.190:41139)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:17.813 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.214.190:41139: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:17.816 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593; will retry
11:45:17.888 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.214.189:36301(127.10.214.189:36301)] connecting to peer
11:45:17.889 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.214.190:41139(127.10.214.190:41139)] connecting to peer
11:45:17.892 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:17.894 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.214.189:36301(127.10.214.189:36301)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:17.893 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:17.896 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.214.190:41139(127.10.214.190:41139)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:17.899 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.214.189:36301: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:17.900 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.214.190:41139: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:17.902 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593; will retry
11:45:18.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:18.082406 11619 catalog_manager.cc:1558] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:45:18.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:18.082667 11619 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:45:18.208 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.214.189:36301(127.10.214.189:36301)] connecting to peer
11:45:18.209 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.214.190:41139(127.10.214.190:41139)] connecting to peer
11:45:18.210 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:18.211 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.214.189:36301(127.10.214.189:36301)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:18.211 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.214.189:36301: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:18.213 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:18.213 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.214.190:41139(127.10.214.190:41139)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:18.213 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.214.190:41139: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:18.216 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593; will retry
11:45:18.608 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.214.189:36301(127.10.214.189:36301)] connecting to peer
11:45:18.610 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.214.190:41139(127.10.214.190:41139)] connecting to peer
11:45:18.611 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:18.611 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.214.189:36301(127.10.214.189:36301)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:18.613 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:18.613 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.214.189:36301: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:18.613 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.214.190:41139(127.10.214.190:41139)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:18.613 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.214.190:41139: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:18.616 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593; will retry
11:45:18.768 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.214.189:36301(127.10.214.189:36301)] connecting to peer
11:45:18.770 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:18.770 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.214.189:36301(127.10.214.189:36301)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:18.770 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.214.189:36301: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:18.770 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.214.190:41139(127.10.214.190:41139)] connecting to peer
11:45:18.776 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:18.776 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.214.190:41139(127.10.214.190:41139)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:18.776 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.214.190:41139: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:18.776 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593; will retry
11:45:19.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:19.085561 11619 catalog_manager.cc:1558] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:45:19.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:19.085834 11619 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:45:19.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:19.701516 11625 raft_consensus.cc:482] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
11:45:19.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:19.701992 11625 raft_consensus.cc:504] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e35b4b67316c413cbdebfb6f5b3c18f3" member_type: VOTER last_known_addr { host: "127.10.214.190" port: 41139 } } peers { permanent_uuid: "aa381d346e1c45879e9319334ad9b2e4" member_type: VOTER last_known_addr { host: "127.10.214.189" port: 36301 } } peers { permanent_uuid: "db14f371e39f4c28a890e1b83b8634ce" member_type: VOTER last_known_addr { host: "127.10.214.188" port: 40593 } }
11:45:19.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:19.703904 11625 leader_election.cc:288] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e35b4b67316c413cbdebfb6f5b3c18f3 (127.10.214.190:41139), aa381d346e1c45879e9319334ad9b2e4 (127.10.214.189:36301)
11:45:19.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:19.709808 11552 leader_election.cc:334] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e35b4b67316c413cbdebfb6f5b3c18f3 (127.10.214.190:41139): Network error: Client connection negotiation failed: client connection to 127.10.214.190:41139: connect: Connection refused (error 111)
11:45:19.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:19.710243 11553 leader_election.cc:334] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer aa381d346e1c45879e9319334ad9b2e4 (127.10.214.189:36301): Network error: Client connection negotiation failed: client connection to 127.10.214.189:36301: connect: Connection refused (error 111)
11:45:19.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:19.710577 11553 leader_election.cc:302] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: db14f371e39f4c28a890e1b83b8634ce; no voters: aa381d346e1c45879e9319334ad9b2e4, e35b4b67316c413cbdebfb6f5b3c18f3
11:45:19.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:19.711089 11625 raft_consensus.cc:2726] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
11:45:20.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:20.088713 11619 catalog_manager.cc:1558] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:45:20.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:20.088970 11619 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:45:21.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:21.091969 11619 catalog_manager.cc:1558] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:45:21.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:21.092203 11619 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:45:21.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:21.740440 11083 debug-util.cc:398] Leaking SignalData structure 0x7b08000b9a40 after lost signal to thread 10958
11:45:21.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:21.741549 11083 debug-util.cc:398] Leaking SignalData structure 0x7b08000bd000 after lost signal to thread 11086
11:45:21.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:21.989967 11632 raft_consensus.cc:482] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
11:45:21.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:21.990484 11632 raft_consensus.cc:504] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e35b4b67316c413cbdebfb6f5b3c18f3" member_type: VOTER last_known_addr { host: "127.10.214.190" port: 41139 } } peers { permanent_uuid: "aa381d346e1c45879e9319334ad9b2e4" member_type: VOTER last_known_addr { host: "127.10.214.189" port: 36301 } } peers { permanent_uuid: "db14f371e39f4c28a890e1b83b8634ce" member_type: VOTER last_known_addr { host: "127.10.214.188" port: 40593 } }
11:45:21.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:21.994474 11632 leader_election.cc:288] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e35b4b67316c413cbdebfb6f5b3c18f3 (127.10.214.190:41139), aa381d346e1c45879e9319334ad9b2e4 (127.10.214.189:36301)
11:45:21.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:21.999059 11552 leader_election.cc:334] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e35b4b67316c413cbdebfb6f5b3c18f3 (127.10.214.190:41139): Network error: Client connection negotiation failed: client connection to 127.10.214.190:41139: connect: Connection refused (error 111)
11:45:21.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:21.999435 11553 leader_election.cc:334] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer aa381d346e1c45879e9319334ad9b2e4 (127.10.214.189:36301): Network error: Client connection negotiation failed: client connection to 127.10.214.189:36301: connect: Connection refused (error 111)
11:45:22.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:21.999888 11553 leader_election.cc:302] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: db14f371e39f4c28a890e1b83b8634ce; no voters: aa381d346e1c45879e9319334ad9b2e4, e35b4b67316c413cbdebfb6f5b3c18f3
11:45:22.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:22.000587 11632 raft_consensus.cc:2726] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
11:45:22.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:22.096338 11619 catalog_manager.cc:1558] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:45:22.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:22.096673 11619 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:45:22.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:22.133781 11532 master_runner.cc:186] Error getting master registration for 127.10.214.190:41139: Network error: Client connection negotiation failed: client connection to 127.10.214.190:41139: connect: Connection refused (error 111),
11:45:22.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:22.134207 11532 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
11:45:22.408 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.214.189:36301(127.10.214.189:36301)] connecting to peer
11:45:22.410 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.214.190:41139(127.10.214.190:41139)] connecting to peer
11:45:22.411 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:22.411 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.214.189:36301(127.10.214.189:36301)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:22.411 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.214.189:36301: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:22.414 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:22.414 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.214.190:41139(127.10.214.190:41139)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:22.414 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.214.190:41139: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:22.417 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593; will retry
11:45:22.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:22.438840 10449 debug-util.cc:398] Leaking SignalData structure 0x7b08000b40a0 after lost signal to thread 10386
11:45:22.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:22.439580 10449 debug-util.cc:398] Leaking SignalData structure 0x7b08000a5320 after lost signal to thread 10452
11:45:22.588 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.214.189:36301(127.10.214.189:36301)] connecting to peer
11:45:22.589 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:22.589 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.214.189:36301(127.10.214.189:36301)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:22.590 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.214.190:41139(127.10.214.190:41139)] connecting to peer
11:45:22.590 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.214.189:36301: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:22.592 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:22.592 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.214.190:41139(127.10.214.190:41139)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:22.592 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.214.190:41139: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:22.595 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593; will retry
11:45:23.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:23.100801 11619 catalog_manager.cc:1558] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:45:23.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:23.101133 11619 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:45:23.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:23.845146 11637 raft_consensus.cc:482] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
11:45:23.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:23.845489 11637 raft_consensus.cc:504] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e35b4b67316c413cbdebfb6f5b3c18f3" member_type: VOTER last_known_addr { host: "127.10.214.190" port: 41139 } } peers { permanent_uuid: "aa381d346e1c45879e9319334ad9b2e4" member_type: VOTER last_known_addr { host: "127.10.214.189" port: 36301 } } peers { permanent_uuid: "db14f371e39f4c28a890e1b83b8634ce" member_type: VOTER last_known_addr { host: "127.10.214.188" port: 40593 } }
11:45:23.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:23.847100 11637 leader_election.cc:288] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e35b4b67316c413cbdebfb6f5b3c18f3 (127.10.214.190:41139), aa381d346e1c45879e9319334ad9b2e4 (127.10.214.189:36301)
11:45:23.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:23.849040 11552 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.214.190:41139: connect: Connection refused (error 111) [suppressed 5 similar messages]
11:45:23.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:23.851683 11553 leader_election.cc:334] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer aa381d346e1c45879e9319334ad9b2e4 (127.10.214.189:36301): Network error: Client connection negotiation failed: client connection to 127.10.214.189:36301: connect: Connection refused (error 111)
11:45:23.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:23.852124 11552 leader_election.cc:334] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e35b4b67316c413cbdebfb6f5b3c18f3 (127.10.214.190:41139): Network error: Client connection negotiation failed: client connection to 127.10.214.190:41139: connect: Connection refused (error 111)
11:45:23.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:23.852375 11552 leader_election.cc:302] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: db14f371e39f4c28a890e1b83b8634ce; no voters: aa381d346e1c45879e9319334ad9b2e4, e35b4b67316c413cbdebfb6f5b3c18f3
11:45:23.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:23.852838 11637 raft_consensus.cc:2726] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
11:45:24.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:24.104063 11619 catalog_manager.cc:1558] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:45:24.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:24.104329 11619 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:45:25.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:25.107072 11619 catalog_manager.cc:1558] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:45:25.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:25.107322 11619 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:45:25.688 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.214.189:36301(127.10.214.189:36301)] connecting to peer
11:45:25.689 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.214.190:41139(127.10.214.190:41139)] connecting to peer
11:45:25.690 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:25.691 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.214.189:36301(127.10.214.189:36301)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:25.691 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.214.189:36301: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:25.693 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:25.693 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.214.190:41139(127.10.214.190:41139)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:25.693 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.214.190:41139: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:25.695 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593; will retry
11:45:26.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:26.110527 11619 catalog_manager.cc:1558] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:45:26.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:26.110865 11619 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:45:26.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:45:26 dist-test-slave-8d15 krb5kdc[11110](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682326, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.214.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:45:26.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:26.756132 11545 init.cc:295] Successfully reacquired a new kerberos TGT
11:45:27.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:27.113803 11619 catalog_manager.cc:1558] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:45:27.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:27.114048 11619 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:45:27.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:27.137408 11532 master_runner.cc:186] Error getting master registration for 127.10.214.190:41139: Network error: Client connection negotiation failed: client connection to 127.10.214.190:41139: connect: Connection refused (error 111),
11:45:27.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:27.137765 11532 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
11:45:27.668 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.214.189:36301(127.10.214.189:36301)] connecting to peer
11:45:27.669 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.214.190:41139(127.10.214.190:41139)] connecting to peer
11:45:27.670 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:27.671 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.214.189:36301(127.10.214.189:36301)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:27.671 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.214.189:36301: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:27.672 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:27.672 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.214.190:41139(127.10.214.190:41139)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:27.672 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.214.190:41139: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:27.674 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593; will retry
11:45:28.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:28.117089 11619 catalog_manager.cc:1558] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:45:28.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:28.117314 11619 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:45:29.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:29.120042 11619 catalog_manager.cc:1558] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:45:29.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:29.120276 11619 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:45:30.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:30.122952 11619 catalog_manager.cc:1558] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:45:30.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:30.123189 11619 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:45:30.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:30.302167 11647 raft_consensus.cc:482] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
11:45:30.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:30.302610 11647 raft_consensus.cc:504] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e35b4b67316c413cbdebfb6f5b3c18f3" member_type: VOTER last_known_addr { host: "127.10.214.190" port: 41139 } } peers { permanent_uuid: "aa381d346e1c45879e9319334ad9b2e4" member_type: VOTER last_known_addr { host: "127.10.214.189" port: 36301 } } peers { permanent_uuid: "db14f371e39f4c28a890e1b83b8634ce" member_type: VOTER last_known_addr { host: "127.10.214.188" port: 40593 } }
11:45:30.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:30.304709 11647 leader_election.cc:288] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e35b4b67316c413cbdebfb6f5b3c18f3 (127.10.214.190:41139), aa381d346e1c45879e9319334ad9b2e4 (127.10.214.189:36301)
11:45:30.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:30.308180 11552 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.214.190:41139: connect: Connection refused (error 111) [suppressed 1 similar messages]
11:45:30.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:30.320394 11552 leader_election.cc:334] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e35b4b67316c413cbdebfb6f5b3c18f3 (127.10.214.190:41139): Network error: Client connection negotiation failed: client connection to 127.10.214.190:41139: connect: Connection refused (error 111)
11:45:30.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:30.321189 11553 leader_election.cc:334] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer aa381d346e1c45879e9319334ad9b2e4 (127.10.214.189:36301): Network error: Client connection negotiation failed: client connection to 127.10.214.189:36301: connect: Connection refused (error 111)
11:45:30.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:30.321609 11553 leader_election.cc:302] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: db14f371e39f4c28a890e1b83b8634ce; no voters: aa381d346e1c45879e9319334ad9b2e4, e35b4b67316c413cbdebfb6f5b3c18f3
11:45:30.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:30.322363 11647 raft_consensus.cc:2726] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
11:45:31.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:31.125990 11619 catalog_manager.cc:1558] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:45:31.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:31.126291 11619 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:45:31.488 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.214.189:36301(127.10.214.189:36301)] connecting to peer
11:45:31.489 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.214.190:41139(127.10.214.190:41139)] connecting to peer
11:45:31.490 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:31.490 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.214.189:36301(127.10.214.189:36301)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:31.491 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.214.189:36301: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:31.492 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:31.492 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.214.190:41139(127.10.214.190:41139)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:31.492 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.214.190:41139: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:31.494 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593; will retry
11:45:32.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:32.130318 11619 catalog_manager.cc:1558] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:45:32.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:32.130649 11619 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:45:32.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:32.140947 11532 master_runner.cc:186] Error getting master registration for 127.10.214.190:41139: Network error: Client connection negotiation failed: client connection to 127.10.214.190:41139: connect: Connection refused (error 111),
11:45:32.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:32.141358 11532 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
11:45:32.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:32.652891 11656 raft_consensus.cc:482] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
11:45:32.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:32.653241 11656 raft_consensus.cc:504] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e35b4b67316c413cbdebfb6f5b3c18f3" member_type: VOTER last_known_addr { host: "127.10.214.190" port: 41139 } } peers { permanent_uuid: "aa381d346e1c45879e9319334ad9b2e4" member_type: VOTER last_known_addr { host: "127.10.214.189" port: 36301 } } peers { permanent_uuid: "db14f371e39f4c28a890e1b83b8634ce" member_type: VOTER last_known_addr { host: "127.10.214.188" port: 40593 } }
11:45:32.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:32.654778 11656 leader_election.cc:288] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e35b4b67316c413cbdebfb6f5b3c18f3 (127.10.214.190:41139), aa381d346e1c45879e9319334ad9b2e4 (127.10.214.189:36301)
11:45:32.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:32.659444 11552 leader_election.cc:334] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e35b4b67316c413cbdebfb6f5b3c18f3 (127.10.214.190:41139): Network error: Client connection negotiation failed: client connection to 127.10.214.190:41139: connect: Connection refused (error 111)
11:45:32.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:32.659932 11553 leader_election.cc:334] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer aa381d346e1c45879e9319334ad9b2e4 (127.10.214.189:36301): Network error: Client connection negotiation failed: client connection to 127.10.214.189:36301: connect: Connection refused (error 111)
11:45:32.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:32.660282 11553 leader_election.cc:302] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: db14f371e39f4c28a890e1b83b8634ce; no voters: aa381d346e1c45879e9319334ad9b2e4, e35b4b67316c413cbdebfb6f5b3c18f3
11:45:32.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:32.660980 11656 raft_consensus.cc:2726] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
11:45:33.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:33.133579 11619 catalog_manager.cc:1558] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:45:33.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:33.133841 11619 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:45:33.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:33.140750 10603 debug-util.cc:398] Leaking SignalData structure 0x7b080003f620 after lost signal to thread 10535
11:45:33.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:33.141276 10603 debug-util.cc:398] Leaking SignalData structure 0x7b080003f420 after lost signal to thread 10606
11:45:34.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:34.137996 11619 catalog_manager.cc:1558] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:45:34.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:34.138345 11619 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:45:34.168 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.214.189:36301(127.10.214.189:36301)] connecting to peer
11:45:34.168 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.214.190:41139(127.10.214.190:41139)] connecting to peer
11:45:34.171 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:34.171 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.214.189:36301(127.10.214.189:36301)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:34.171 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.214.189:36301: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:34.173 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:34.173 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.214.190:41139(127.10.214.190:41139)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:34.173 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.214.190:41139: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:34.175 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593; will retry
11:45:35.087 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.214.189:36301(127.10.214.189:36301)] connecting to peer
11:45:35.089 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.214.190:41139(127.10.214.190:41139)] connecting to peer
11:45:35.090 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:35.090 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.214.189:36301(127.10.214.189:36301)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:35.090 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.214.189:36301: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:35.091 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:35.091 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.214.190:41139(127.10.214.190:41139)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:35.092 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.214.190:41139: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:35.093 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593; will retry
11:45:35.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:35.141330 11619 catalog_manager.cc:1558] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:45:35.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:35.141583 11619 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:45:36.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:36.145035 11619 catalog_manager.cc:1558] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:45:36.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:36.145347 11619 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:45:36.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:45:36 dist-test-slave-8d15 krb5kdc[11110](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682336, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.214.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:45:36.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:36.758385 11545 init.cc:295] Successfully reacquired a new kerberos TGT
11:45:37.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:37.144415 11532 master_runner.cc:186] Error getting master registration for 127.10.214.190:41139: Network error: Client connection negotiation failed: client connection to 127.10.214.190:41139: connect: Connection refused (error 111),
11:45:37.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:37.144764 11532 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
11:45:37.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:37.147905 11619 catalog_manager.cc:1558] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:45:37.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:37.148099 11619 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:45:38.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:38.150997 11619 catalog_manager.cc:1558] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:45:38.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:38.151223 11619 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:45:38.587 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.214.189:36301(127.10.214.189:36301)] connecting to peer
11:45:38.589 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.214.190:41139(127.10.214.190:41139)] connecting to peer
11:45:38.589 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:38.590 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.214.189:36301(127.10.214.189:36301)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:38.590 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.214.189:36301: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:38.591 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:38.591 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.214.190:41139(127.10.214.190:41139)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:38.591 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.214.190:41139: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:38.593 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593; will retry
11:45:38.628 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.214.189:36301(127.10.214.189:36301)] connecting to peer
11:45:38.629 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.214.190:41139(127.10.214.190:41139)] connecting to peer
11:45:38.630 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:38.630 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.214.189:36301(127.10.214.189:36301)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:38.630 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.214.189:36301: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:38.631 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:38.631 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.214.190:41139(127.10.214.190:41139)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:38.632 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.214.190:41139: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:38.634 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593; will retry
11:45:38.748 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.214.189:36301(127.10.214.189:36301)] connecting to peer
11:45:38.749 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.214.190:41139(127.10.214.190:41139)] connecting to peer
11:45:38.749 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:38.750 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.214.189:36301(127.10.214.189:36301)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:38.750 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.214.189:36301: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:38.751 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:38.751 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.214.190:41139(127.10.214.190:41139)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:38.751 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.214.190:41139: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:38.753 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593; will retry
11:45:39.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:39.153831 11619 catalog_manager.cc:1558] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:45:39.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:39.154103 11619 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:45:40.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:40.157260 11619 catalog_manager.cc:1558] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:45:40.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:40.157509 11619 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:45:41.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:41.160461 11619 catalog_manager.cc:1558] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:45:41.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:41.160715 11619 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:45:41.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:41.666011 11672 raft_consensus.cc:482] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
11:45:41.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:41.666445 11672 raft_consensus.cc:504] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e35b4b67316c413cbdebfb6f5b3c18f3" member_type: VOTER last_known_addr { host: "127.10.214.190" port: 41139 } } peers { permanent_uuid: "aa381d346e1c45879e9319334ad9b2e4" member_type: VOTER last_known_addr { host: "127.10.214.189" port: 36301 } } peers { permanent_uuid: "db14f371e39f4c28a890e1b83b8634ce" member_type: VOTER last_known_addr { host: "127.10.214.188" port: 40593 } }
11:45:41.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:41.668217 11672 leader_election.cc:288] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e35b4b67316c413cbdebfb6f5b3c18f3 (127.10.214.190:41139), aa381d346e1c45879e9319334ad9b2e4 (127.10.214.189:36301)
11:45:41.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:41.670056 11552 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.214.190:41139: connect: Connection refused (error 111) [suppressed 3 similar messages]
11:45:41.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:41.673099 11553 leader_election.cc:334] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer aa381d346e1c45879e9319334ad9b2e4 (127.10.214.189:36301): Network error: Client connection negotiation failed: client connection to 127.10.214.189:36301: connect: Connection refused (error 111)
11:45:41.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:41.673581 11552 leader_election.cc:334] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e35b4b67316c413cbdebfb6f5b3c18f3 (127.10.214.190:41139): Network error: Client connection negotiation failed: client connection to 127.10.214.190:41139: connect: Connection refused (error 111)
11:45:41.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:41.673906 11552 leader_election.cc:302] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: db14f371e39f4c28a890e1b83b8634ce; no voters: aa381d346e1c45879e9319334ad9b2e4, e35b4b67316c413cbdebfb6f5b3c18f3
11:45:41.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:41.674531 11672 raft_consensus.cc:2726] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
11:45:41.988 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.214.189:36301(127.10.214.189:36301)] connecting to peer
11:45:41.989 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.214.190:41139(127.10.214.190:41139)] connecting to peer
11:45:41.989 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:41.990 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.214.189:36301(127.10.214.189:36301)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:41.990 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.214.189:36301: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:41.990 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:41.990 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.214.190:41139(127.10.214.190:41139)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:41.991 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.214.190:41139: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:41.993 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593; will retry
11:45:42.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:42.146523 11532 master_runner.cc:186] Error getting master registration for 127.10.214.190:41139: Network error: Client connection negotiation failed: client connection to 127.10.214.190:41139: connect: Connection refused (error 111),
11:45:42.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:42.146822 11532 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
11:45:42.148 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.214.189:36301(127.10.214.189:36301)] connecting to peer
11:45:42.148 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.214.190:41139(127.10.214.190:41139)] connecting to peer
11:45:42.149 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:42.149 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.214.189:36301(127.10.214.189:36301)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:42.149 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.214.189:36301: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:42.150 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:42.150 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.214.190:41139(127.10.214.190:41139)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:42.151 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.214.190:41139: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:42.152 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593; will retry
11:45:42.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:42.164068 11619 catalog_manager.cc:1558] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:45:42.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:42.164332 11619 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:45:43.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:43.166967 11619 catalog_manager.cc:1558] T 00000000000000000000000000000000 P db14f371e39f4c28a890e1b83b8634ce: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:45:43.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:45:43.167189 11619 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:45:43.228 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.214.189:36301(127.10.214.189:36301)] connecting to peer
11:45:43.229 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.214.190:41139(127.10.214.190:41139)] connecting to peer
11:45:43.229 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:43.229 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.214.189:36301(127.10.214.189:36301)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:43.230 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.214.189:36301: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:43.230 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:43.230 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.214.190:41139(127.10.214.190:41139)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:43.231 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.214.190:41139: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:43.233 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593; will retry
11:45:43.448 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.214.189:36301(127.10.214.189:36301)] connecting to peer
11:45:43.449 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.214.190:41139(127.10.214.190:41139)] connecting to peer
11:45:43.450 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:43.450 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.214.189:36301(127.10.214.189:36301)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:43.450 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.214.189:36301: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301
11:45:43.451 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:43.451 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.214.190:41139(127.10.214.190:41139)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:43.451 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.214.190:41139: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
11:45:43.454 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593; will retry
11:45:43.458 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=26548), Traces: [0ms] refreshing cache from master, [36ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.189:36301, [109ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.190:41139, [113ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.188:40593, [169ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.190:41139: Network error: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [168ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.189:36301: Network error: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [590ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.188:40593: OK, [595ms] delaying RPC due to: Service unavailable: Master config (127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [615ms] refreshing cache from master, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.189:36301, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.190:41139, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.188:40593, [625ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.189:36301: Network error: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [628ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.190:41139: Network error: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [630ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.188:40593: OK, [632ms] delaying RPC due to: Service unavailable: Master config (127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [635ms] refreshing cache from master, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.189:36301, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.190:41139, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.188:40593, [638ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.189:36301: Network error: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [640ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.190:41139: Network error: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [642ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.188:40593: OK, [643ms] delaying RPC due to: Service unavailable: Master config (127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [656ms] refreshing cache from master, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.189:36301, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.190:41139, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.188:40593, [659ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.189:36301: Network error: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [662ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.190:41139: Network error: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [664ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.188:40593: OK, [665ms] delaying RPC due to: Service unavailable: Master config (127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [695ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.189:36301, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.190:41139, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.188:40593, [699ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.189:36301: Network error: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [705ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.188:40593: OK, [706ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.190:41139: Network error: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [708ms] delaying RPC due to: Service unavailable: Master config (127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [736ms] refreshing cache from master, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.189:36301, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.190:41139, [739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.188:40593, [740ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.189:36301: Network error: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [743ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.190:41139: Network error: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [744ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.188:40593: OK, [745ms] delaying RPC due to: Service unavailable: Master config (127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [775ms] refreshing cache from master, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.189:36301, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.190:41139, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.188:40593, [779ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.189:36301: Network error: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [781ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.190:41139: Network error: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [783ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.188:40593: OK, [784ms] delaying RPC due to: Service unavailable: Master config (127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [895ms] refreshing cache from master, [895ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.189:36301, [897ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.190:41139, [898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.188:40593, [899ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.189:36301: Network error: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [901ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.190:41139: Network error: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [903ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.188:40593: OK, [904ms] delaying RPC due to: Service unavailable: Master config (127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [976ms] refreshing cache from master, [976ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.189:36301, [977ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.190:41139, [978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.188:40593, [983ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.189:36301: Network error: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [987ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.190:41139: Network error: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [989ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.188:40593: OK, [990ms] delaying RPC due to: Service unavailable: Master config (127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [1295ms] refreshing cache from master, [1296ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.189:36301, [1297ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.190:41139, [1298ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.188:40593, [1299ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.189:36301: Network error: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [1301ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.190:41139: Network error: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [1303ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.188:40593: OK, [1304ms] delaying RPC due to: Service unavailable: Master config (127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [1696ms] refreshing cache from master, [1696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.189:36301, [1698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.190:41139, [1699ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.188:40593, [1699ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.189:36301: Network error: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [1701ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.190:41139: Network error: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [1703ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.188:40593: OK, [1704ms] delaying RPC due to: Service unavailable: Master config (127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [1855ms] refreshing cache from master, [1855ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.189:36301, [1857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.190:41139, [1858ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.189:36301: Network error: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [1859ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.188:40593, [1863ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.188:40593: OK, [1864ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.190:41139: Network error: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [1865ms] delaying RPC due to: Service unavailable: Master config (127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [5496ms] refreshing cache from master, [5496ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.189:36301, [5497ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.190:41139, [5499ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.189:36301: Network error: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [5500ms] too many traces: truncated at 100 traces, deferred=Deferred@1849646869(state=PENDING, result=null, callback=(continuation of Deferred@28832196 after retry RPC after error@344756087) -> (continuation of Deferred@1226598457 after retry RPC after error@1573863928) -> (continuation of Deferred@1674097236 after retry RPC after error@323972717) -> (continuation of Deferred@960004509 after retry RPC after error@509597635) -> (continuation of Deferred@950246478 after retry RPC after error@1350021425) -> (continuation of Deferred@759097035 after retry RPC after error@1296026976) -> (continuation of Deferred@1775925605 after retry RPC after error@80327510) -> (continuation of Deferred@1578833338 after retry RPC after error@1836877780) -> (continuation of Deferred@1473759823 after retry RPC after error@508395744) -> (continuation of Deferred@410906710 after retry RPC after error@297174698) -> (continuation of Deferred@2019210081 after retry RPC after error@174224906) -> (continuation of Deferred@2112538704 after retry RPC after error@1489801873) -> (continuation of Deferred@1973482322 after retry RPC after error@1818954186) -> (continuation of Deferred@1206445151 after retry RPC after error@2064428585) -> (continuation of Deferred@155510905 after retry RPC after error@1029014978) -> (continuation of Deferred@1033847851 after retry RPC after error@595380311) -> (continuation of Deferred@232123233 after retry RPC after error@482284684) -> (continuation of Deferred@1531356460 after retry RPC after error@515764323) -> (continuation of Deferred@1007787574 after retry RPC after error@1483667020) -> (continuation of Deferred@921655208 after retry RPC after error@1193318398) -> (continuation of Deferred@2003849474 after retry RPC after error@1484981558) -> (continuation of Deferred@337468313 after retry RPC after error@1472307587) -> (continuation of Deferred@575410744 after retry RPC after error@1646660082) -> (continuation of Deferred@471418812 after retry RPC after error@1183723525) -> (continuation of Deferred@68237118 after retry RPC after error@1454032855), errback=(continuation of Deferred@28832196 after retry RPC after error@344756087) -> (continuation of Deferred@1226598457 after retry RPC after error@1573863928) -> (continuation of Deferred@1674097236 after retry RPC after error@323972717) -> (continuation of Deferred@960004509 after retry RPC after error@509597635) -> (continuation of Deferred@950246478 after retry RPC after error@1350021425) -> (continuation of Deferred@759097035 after retry RPC after error@1296026976) -> (continuation of Deferred@1775925605 after retry RPC after error@80327510) -> (continuation of Deferred@1578833338 after retry RPC after error@1836877780) -> (continuation of Deferred@1473759823 after retry RPC after error@508395744) -> (continuation of Deferred@410906710 after retry RPC after error@297174698) -> (continuation of Deferred@2019210081 after retry RPC after error@174224906) -> (continuation of Deferred@2112538704 after retry RPC after error@1489801873) -> (continuation of Deferred@1973482322 after retry RPC after error@1818954186) -> (continuation of Deferred@1206445151 after retry RPC after error@2064428585) -> (continuation of Deferred@155510905 after retry RPC after error@1029014978) -> (continuation of Deferred@1033847851 after retry RPC after error@595380311) -> (continuation of Deferred@232123233 after retry RPC after error@482284684) -> (continuation of Deferred@1531356460 after retry RPC after error@515764323) -> (continuation of Deferred@1007787574 after retry RPC after error@1483667020) -> (continuation of Deferred@921655208 after retry RPC after error@1193318398) -> (continuation of Deferred@2003849474 after retry RPC after error@1484981558) -> (continuation of Deferred@337468313 after retry RPC after error@1472307587) -> (continuation of Deferred@575410744 after retry RPC after error@1646660082) -> (continuation of Deferred@471418812 after retry RPC after error@1183723525) -> (continuation of Deferred@68237118 after retry RPC after error@1454032855)))
11:45:43.463 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.10.214.189:36301(127.10.214.189:36301)] disconnecting while in state TERMINATED
11:45:43.464 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.10.214.188:40593(127.10.214.188:40593)] disconnecting while in state READY
11:45:43.464 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.10.214.190:41139(127.10.214.190:41139)] disconnecting while in state TERMINATED
11:45:43.469 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
11:45:43.472 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
11:45:43.474 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.10.214.188:40593(127.10.214.188:40593)] handling channelInactive
11:45:43.474 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.214.188:40593(127.10.214.188:40593)] cleaning up while in state READY due to: connection closed
11:45:43.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:43.477221 11098 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskst9njx/build/tsan/bin/kudu with pid 11532
11:45:44.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T11:45:44Z chronyd exiting
11:45:45.666 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
11:45:45.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:45.667344 10373 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskst9njx/build/tsan/bin/kudu with pid 10654
11:45:45.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:45.691264 10373 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskst9njx/build/tsan/bin/kudu with pid 10813
11:45:45.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:45.718287 10373 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskst9njx/build/tsan/bin/kudu with pid 10954
11:45:45.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:45.744259 10373 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskst9njx/build/tsan/bin/kudu with pid 10385
11:45:45.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:45.774308 10373 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskst9njx/build/tsan/bin/kudu with pid 10455
11:45:45.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:45:45.803838 10373 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskst9njx/build/tsan/bin/kudu with pid 10530
11:45:46.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T11:45:46Z chronyd exiting
11:45:47.963 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=26548), Traces: [0ms] refreshing cache from master, [36ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.189:36301, [109ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.190:41139, [113ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.188:40593, [169ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.190:41139: Network error: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [168ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.189:36301: Network error: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [590ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.188:40593: OK, [595ms] delaying RPC due to: Service unavailable: Master config (127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [615ms] refreshing cache from master, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.189:36301, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.190:41139, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.188:40593, [625ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.189:36301: Network error: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [628ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.190:41139: Network error: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [630ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.188:40593: OK, [632ms] delaying RPC due to: Service unavailable: Master config (127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [635ms] refreshing cache from master, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.189:36301, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.190:41139, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.188:40593, [638ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.189:36301: Network error: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [640ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.190:41139: Network error: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [642ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.188:40593: OK, [643ms] delaying RPC due to: Service unavailable: Master config (127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [656ms] refreshing cache from master, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.189:36301, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.190:41139, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.188:40593, [659ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.189:36301: Network error: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [662ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.190:41139: Network error: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [664ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.188:40593: OK, [665ms] delaying RPC due to: Service unavailable: Master config (127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [695ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.189:36301, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.190:41139, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.188:40593, [699ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.189:36301: Network error: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [705ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.188:40593: OK, [706ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.190:41139: Network error: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [708ms] delaying RPC due to: Service unavailable: Master config (127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [736ms] refreshing cache from master, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.189:36301, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.190:41139, [739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.188:40593, [740ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.189:36301: Network error: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [743ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.190:41139: Network error: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [744ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.188:40593: OK, [745ms] delaying RPC due to: Service unavailable: Master config (127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [775ms] refreshing cache from master, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.189:36301, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.190:41139, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.188:40593, [779ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.189:36301: Network error: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [781ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.190:41139: Network error: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [783ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.188:40593: OK, [784ms] delaying RPC due to: Service unavailable: Master config (127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [895ms] refreshing cache from master, [895ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.189:36301, [897ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.190:41139, [898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.188:40593, [899ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.189:36301: Network error: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [901ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.190:41139: Network error: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [903ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.188:40593: OK, [904ms] delaying RPC due to: Service unavailable: Master config (127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [976ms] refreshing cache from master, [976ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.189:36301, [977ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.190:41139, [978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.188:40593, [983ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.189:36301: Network error: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [987ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.190:41139: Network error: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [989ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.188:40593: OK, [990ms] delaying RPC due to: Service unavailable: Master config (127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [1295ms] refreshing cache from master, [1296ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.189:36301, [1297ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.190:41139, [1298ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.188:40593, [1299ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.189:36301: Network error: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [1301ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.190:41139: Network error: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [1303ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.188:40593: OK, [1304ms] delaying RPC due to: Service unavailable: Master config (127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [1696ms] refreshing cache from master, [1696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.189:36301, [1698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.190:41139, [1699ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.188:40593, [1699ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.189:36301: Network error: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [1701ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.190:41139: Network error: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [1703ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.188:40593: OK, [1704ms] delaying RPC due to: Service unavailable: Master config (127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [1855ms] refreshing cache from master, [1855ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.189:36301, [1857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.190:41139, [1858ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.189:36301: Network error: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [1859ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.188:40593, [1863ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.188:40593: OK, [1864ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.190:41139: Network error: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [1865ms] delaying RPC due to: Service unavailable: Master config (127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [5496ms] refreshing cache from master, [5496ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.189:36301, [5497ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.190:41139, [5499ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.189:36301: Network error: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [5500ms] too many traces: truncated at 100 traces, deferred=Deferred@1849646869(state=PENDING, result=null, callback=(continuation of Deferred@28832196 after retry RPC after error@344756087) -> (continuation of Deferred@1226598457 after retry RPC after error@1573863928) -> (continuation of Deferred@1674097236 after retry RPC after error@323972717) -> (continuation of Deferred@960004509 after retry RPC after error@509597635) -> (continuation of Deferred@950246478 after retry RPC after error@1350021425) -> (continuation of Deferred@759097035 after retry RPC after error@1296026976) -> (continuation of Deferred@1775925605 after retry RPC after error@80327510) -> (continuation of Deferred@1578833338 after retry RPC after error@1836877780) -> (continuation of Deferred@1473759823 after retry RPC after error@508395744) -> (continuation of Deferred@410906710 after retry RPC after error@297174698) -> (continuation of Deferred@2019210081 after retry RPC after error@174224906) -> (continuation of Deferred@2112538704 after retry RPC after error@1489801873) -> (continuation of Deferred@1973482322 after retry RPC after error@1818954186) -> (continuation of Deferred@1206445151 after retry RPC after error@2064428585) -> (continuation of Deferred@155510905 after retry RPC after error@1029014978) -> (continuation of Deferred@1033847851 after retry RPC after error@595380311) -> (continuation of Deferred@232123233 after retry RPC after error@482284684) -> (continuation of Deferred@1531356460 after retry RPC after error@515764323) -> (continuation of Deferred@1007787574 after retry RPC after error@1483667020) -> (continuation of Deferred@921655208 after retry RPC after error@1193318398) -> (continuation of Deferred@2003849474 after retry RPC after error@1484981558) -> (continuation of Deferred@337468313 after retry RPC after error@1472307587) -> (continuation of Deferred@575410744 after retry RPC after error@1646660082) -> (continuation of Deferred@471418812 after retry RPC after error@1183723525) -> (continuation of Deferred@68237118 after retry RPC after error@1454032855), errback=(continuation of Deferred@28832196 after retry RPC after error@344756087) -> (continuation of Deferred@1226598457 after retry RPC after error@1573863928) -> (continuation of Deferred@1674097236 after retry RPC after error@323972717) -> (continuation of Deferred@960004509 after retry RPC after error@509597635) -> (continuation of Deferred@950246478 after retry RPC after error@1350021425) -> (continuation of Deferred@759097035 after retry RPC after error@1296026976) -> (continuation of Deferred@1775925605 after retry RPC after error@80327510) -> (continuation of Deferred@1578833338 after retry RPC after error@1836877780) -> (continuation of Deferred@1473759823 after retry RPC after error@508395744) -> (continuation of Deferred@410906710 after retry RPC after error@297174698) -> (continuation of Deferred@2019210081 after retry RPC after error@174224906) -> (continuation of Deferred@2112538704 after retry RPC after error@1489801873) -> (continuation of Deferred@1973482322 after retry RPC after error@1818954186) -> (continuation of Deferred@1206445151 after retry RPC after error@2064428585) -> (continuation of Deferred@155510905 after retry RPC after error@1029014978) -> (continuation of Deferred@1033847851 after retry RPC after error@595380311) -> (continuation of Deferred@232123233 after retry RPC after error@482284684) -> (continuation of Deferred@1531356460 after retry RPC after error@515764323) -> (continuation of Deferred@1007787574 after retry RPC after error@1483667020) -> (continuation of Deferred@921655208 after retry RPC after error@1193318398) -> (continuation of Deferred@2003849474 after retry RPC after error@1484981558) -> (continuation of Deferred@337468313 after retry RPC after error@1472307587) -> (continuation of Deferred@575410744 after retry RPC after error@1646660082) -> (continuation of Deferred@471418812 after retry RPC after error@1183723525) -> (continuation of Deferred@68237118 after retry RPC after error@1454032855)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.access$200(DefaultPromise.java:35) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise$1.run(DefaultPromise.java:503) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.10.214.190:41139
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=26548), Traces: [0ms] refreshing cache from master, [36ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.189:36301, [109ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.190:41139, [113ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.188:40593, [169ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.190:41139: Network error: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [168ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.189:36301: Network error: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [590ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.188:40593: OK, [595ms] delaying RPC due to: Service unavailable: Master config (127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [615ms] refreshing cache from master, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.189:36301, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.190:41139, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.188:40593, [625ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.189:36301: Network error: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [628ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.190:41139: Network error: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [630ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.188:40593: OK, [632ms] delaying RPC due to: Service unavailable: Master config (127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [635ms] refreshing cache from master, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.189:36301, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.190:41139, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.188:40593, [638ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.189:36301: Network error: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [640ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.190:41139: Network error: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [642ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.188:40593: OK, [643ms] delaying RPC due to: Service unavailable: Master config (127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [656ms] refreshing cache from master, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.189:36301, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.190:41139, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.188:40593, [659ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.189:36301: Network error: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [662ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.190:41139: Network error: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [664ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.188:40593: OK, [665ms] delaying RPC due to: Service unavailable: Master config (127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [695ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.189:36301, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.190:41139, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.188:40593, [699ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.189:36301: Network error: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [705ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.188:40593: OK, [706ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.190:41139: Network error: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [708ms] delaying RPC due to: Service unavailable: Master config (127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [736ms] refreshing cache from master, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.189:36301, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.190:41139, [739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.188:40593, [740ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.189:36301: Network error: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [743ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.190:41139: Network error: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [744ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.188:40593: OK, [745ms] delaying RPC due to: Service unavailable: Master config (127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [775ms] refreshing cache from master, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.189:36301, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.190:41139, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.188:40593, [779ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.189:36301: Network error: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [781ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.190:41139: Network error: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [783ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.188:40593: OK, [784ms] delaying RPC due to: Service unavailable: Master config (127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [895ms] refreshing cache from master, [895ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.189:36301, [897ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.190:41139, [898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.188:40593, [899ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.189:36301: Network error: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [901ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.190:41139: Network error: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [903ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.188:40593: OK, [904ms] delaying RPC due to: Service unavailable: Master config (127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [976ms] refreshing cache from master, [976ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.189:36301, [977ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.190:41139, [978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.188:40593, [983ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.189:36301: Network error: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [987ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.190:41139: Network error: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [989ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.188:40593: OK, [990ms] delaying RPC due to: Service unavailable: Master config (127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [1295ms] refreshing cache from master, [1296ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.189:36301, [1297ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.190:41139, [1298ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.188:40593, [1299ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.189:36301: Network error: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [1301ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.190:41139: Network error: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [1303ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.188:40593: OK, [1304ms] delaying RPC due to: Service unavailable: Master config (127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [1696ms] refreshing cache from master, [1696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.189:36301, [1698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.190:41139, [1699ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.188:40593, [1699ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.189:36301: Network error: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [1701ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.190:41139: Network error: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [1703ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.188:40593: OK, [1704ms] delaying RPC due to: Service unavailable: Master config (127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [1855ms] refreshing cache from master, [1855ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.189:36301, [1857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.190:41139, [1858ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.189:36301: Network error: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [1859ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.188:40593, [1863ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.188:40593: OK, [1864ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.190:41139: Network error: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [1865ms] delaying RPC due to: Service unavailable: Master config (127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [5496ms] refreshing cache from master, [5496ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.189:36301, [5497ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.190:41139, [5499ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.189:36301: Network error: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [5500ms] too many traces: truncated at 100 traces, deferred=Deferred@1849646869(state=PENDING, result=null, callback=(continuation of Deferred@28832196 after retry RPC after error@344756087) -> (continuation of Deferred@1226598457 after retry RPC after error@1573863928) -> (continuation of Deferred@1674097236 after retry RPC after error@323972717) -> (continuation of Deferred@960004509 after retry RPC after error@509597635) -> (continuation of Deferred@950246478 after retry RPC after error@1350021425) -> (continuation of Deferred@759097035 after retry RPC after error@1296026976) -> (continuation of Deferred@1775925605 after retry RPC after error@80327510) -> (continuation of Deferred@1578833338 after retry RPC after error@1836877780) -> (continuation of Deferred@1473759823 after retry RPC after error@508395744) -> (continuation of Deferred@410906710 after retry RPC after error@297174698) -> (continuation of Deferred@2019210081 after retry RPC after error@174224906) -> (continuation of Deferred@2112538704 after retry RPC after error@1489801873) -> (continuation of Deferred@1973482322 after retry RPC after error@1818954186) -> (continuation of Deferred@1206445151 after retry RPC after error@2064428585) -> (continuation of Deferred@155510905 after retry RPC after error@1029014978) -> (continuation of Deferred@1033847851 after retry RPC after error@595380311) -> (continuation of Deferred@232123233 after retry RPC after error@482284684) -> (continuation of Deferred@1531356460 after retry RPC after error@515764323) -> (continuation of Deferred@1007787574 after retry RPC after error@1483667020) -> (continuation of Deferred@921655208 after retry RPC after error@1193318398) -> (continuation of Deferred@2003849474 after retry RPC after error@1484981558) -> (continuation of Deferred@337468313 after retry RPC after error@1472307587) -> (continuation of Deferred@575410744 after retry RPC after error@1646660082) -> (continuation of Deferred@471418812 after retry RPC after error@1183723525) -> (continuation of Deferred@68237118 after retry RPC after error@1454032855), errback=(continuation of Deferred@28832196 after retry RPC after error@344756087) -> (continuation of Deferred@1226598457 after retry RPC after error@1573863928) -> (continuation of Deferred@1674097236 after retry RPC after error@323972717) -> (continuation of Deferred@960004509 after retry RPC after error@509597635) -> (continuation of Deferred@950246478 after retry RPC after error@1350021425) -> (continuation of Deferred@759097035 after retry RPC after error@1296026976) -> (continuation of Deferred@1775925605 after retry RPC after error@80327510) -> (continuation of Deferred@1578833338 after retry RPC after error@1836877780) -> (continuation of Deferred@1473759823 after retry RPC after error@508395744) -> (continuation of Deferred@410906710 after retry RPC after error@297174698) -> (continuation of Deferred@2019210081 after retry RPC after error@174224906) -> (continuation of Deferred@2112538704 after retry RPC after error@1489801873) -> (continuation of Deferred@1973482322 after retry RPC after error@1818954186) -> (continuation of Deferred@1206445151 after retry RPC after error@2064428585) -> (continuation of Deferred@155510905 after retry RPC after error@1029014978) -> (continuation of Deferred@1033847851 after retry RPC after error@595380311) -> (continuation of Deferred@232123233 after retry RPC after error@482284684) -> (continuation of Deferred@1531356460 after retry RPC after error@515764323) -> (continuation of Deferred@1007787574 after retry RPC after error@1483667020) -> (continuation of Deferred@921655208 after retry RPC after error@1193318398) -> (continuation of Deferred@2003849474 after retry RPC after error@1484981558) -> (continuation of Deferred@337468313 after retry RPC after error@1472307587) -> (continuation of Deferred@575410744 after retry RPC after error@1646660082) -> (continuation of Deferred@471418812 after retry RPC after error@1183723525) -> (continuation of Deferred@68237118 after retry RPC after error@1454032855)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557)
at io.netty.util.concurrent.DefaultPromise.access$200(DefaultPromise.java:35)
at io.netty.util.concurrent.DefaultPromise$1.run(DefaultPromise.java:503)
at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174)
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167)
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569)
... 5 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.10.214.190:41139
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
11:45:48.863 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
11:45:48.863 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=26548), Traces: [0ms] refreshing cache from master, [36ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.189:36301, [109ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.190:41139, [113ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.188:40593, [169ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.190:41139: Network error: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [168ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.189:36301: Network error: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [590ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.188:40593: OK, [595ms] delaying RPC due to: Service unavailable: Master config (127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [615ms] refreshing cache from master, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.189:36301, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.190:41139, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.188:40593, [625ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.189:36301: Network error: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [628ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.190:41139: Network error: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [630ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.188:40593: OK, [632ms] delaying RPC due to: Service unavailable: Master config (127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [635ms] refreshing cache from master, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.189:36301, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.190:41139, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.188:40593, [638ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.189:36301: Network error: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [640ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.190:41139: Network error: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [642ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.188:40593: OK, [643ms] delaying RPC due to: Service unavailable: Master config (127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [656ms] refreshing cache from master, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.189:36301, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.190:41139, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.188:40593, [659ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.189:36301: Network error: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [662ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.190:41139: Network error: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [664ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.188:40593: OK, [665ms] delaying RPC due to: Service unavailable: Master config (127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [695ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.189:36301, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.190:41139, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.188:40593, [699ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.189:36301: Network error: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [705ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.188:40593: OK, [706ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.190:41139: Network error: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [708ms] delaying RPC due to: Service unavailable: Master config (127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [736ms] refreshing cache from master, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.189:36301, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.190:41139, [739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.188:40593, [740ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.189:36301: Network error: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [743ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.190:41139: Network error: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [744ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.188:40593: OK, [745ms] delaying RPC due to: Service unavailable: Master config (127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [775ms] refreshing cache from master, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.189:36301, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.190:41139, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.188:40593, [779ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.189:36301: Network error: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [781ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.190:41139: Network error: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [783ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.188:40593: OK, [784ms] delaying RPC due to: Service unavailable: Master config (127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [895ms] refreshing cache from master, [895ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.189:36301, [897ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.190:41139, [898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.188:40593, [899ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.189:36301: Network error: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [901ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.190:41139: Network error: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [903ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.188:40593: OK, [904ms] delaying RPC due to: Service unavailable: Master config (127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [976ms] refreshing cache from master, [976ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.189:36301, [977ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.190:41139, [978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.188:40593, [983ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.189:36301: Network error: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [987ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.190:41139: Network error: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [989ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.188:40593: OK, [990ms] delaying RPC due to: Service unavailable: Master config (127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [1295ms] refreshing cache from master, [1296ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.189:36301, [1297ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.190:41139, [1298ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.188:40593, [1299ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.189:36301: Network error: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [1301ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.190:41139: Network error: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [1303ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.188:40593: OK, [1304ms] delaying RPC due to: Service unavailable: Master config (127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [1696ms] refreshing cache from master, [1696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.189:36301, [1698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.190:41139, [1699ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.188:40593, [1699ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.189:36301: Network error: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [1701ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.190:41139: Network error: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [1703ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.188:40593: OK, [1704ms] delaying RPC due to: Service unavailable: Master config (127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [1855ms] refreshing cache from master, [1855ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.189:36301, [1857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.190:41139, [1858ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.189:36301: Network error: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [1859ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.188:40593, [1863ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.188:40593: OK, [1864ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.190:41139: Network error: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [1865ms] delaying RPC due to: Service unavailable: Master config (127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [5496ms] refreshing cache from master, [5496ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.189:36301, [5497ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.190:41139, [5499ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.189:36301: Network error: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [5500ms] too many traces: truncated at 100 traces, deferred=Deferred@1849646869(state=PENDING, result=null, callback=(continuation of Deferred@28832196 after retry RPC after error@344756087) -> (continuation of Deferred@1226598457 after retry RPC after error@1573863928) -> (continuation of Deferred@1674097236 after retry RPC after error@323972717) -> (continuation of Deferred@960004509 after retry RPC after error@509597635) -> (continuation of Deferred@950246478 after retry RPC after error@1350021425) -> (continuation of Deferred@759097035 after retry RPC after error@1296026976) -> (continuation of Deferred@1775925605 after retry RPC after error@80327510) -> (continuation of Deferred@1578833338 after retry RPC after error@1836877780) -> (continuation of Deferred@1473759823 after retry RPC after error@508395744) -> (continuation of Deferred@410906710 after retry RPC after error@297174698) -> (continuation of Deferred@2019210081 after retry RPC after error@174224906) -> (continuation of Deferred@2112538704 after retry RPC after error@1489801873) -> (continuation of Deferred@1973482322 after retry RPC after error@1818954186) -> (continuation of Deferred@1206445151 after retry RPC after error@2064428585) -> (continuation of Deferred@155510905 after retry RPC after error@1029014978) -> (continuation of Deferred@1033847851 after retry RPC after error@595380311) -> (continuation of Deferred@232123233 after retry RPC after error@482284684) -> (continuation of Deferred@1531356460 after retry RPC after error@515764323) -> (continuation of Deferred@1007787574 after retry RPC after error@1483667020) -> (continuation of Deferred@921655208 after retry RPC after error@1193318398) -> (continuation of Deferred@2003849474 after retry RPC after error@1484981558) -> (continuation of Deferred@337468313 after retry RPC after error@1472307587) -> (continuation of Deferred@575410744 after retry RPC after error@1646660082) -> (continuation of Deferred@471418812 after retry RPC after error@1183723525) -> (continuation of Deferred@68237118 after retry RPC after error@1454032855), errback=(continuation of Deferred@28832196 after retry RPC after error@344756087) -> (continuation of Deferred@1226598457 after retry RPC after error@1573863928) -> (continuation of Deferred@1674097236 after retry RPC after error@323972717) -> (continuation of Deferred@960004509 after retry RPC after error@509597635) -> (continuation of Deferred@950246478 after retry RPC after error@1350021425) -> (continuation of Deferred@759097035 after retry RPC after error@1296026976) -> (continuation of Deferred@1775925605 after retry RPC after error@80327510) -> (continuation of Deferred@1578833338 after retry RPC after error@1836877780) -> (continuation of Deferred@1473759823 after retry RPC after error@508395744) -> (continuation of Deferred@410906710 after retry RPC after error@297174698) -> (continuation of Deferred@2019210081 after retry RPC after error@174224906) -> (continuation of Deferred@2112538704 after retry RPC after error@1489801873) -> (continuation of Deferred@1973482322 after retry RPC after error@1818954186) -> (continuation of Deferred@1206445151 after retry RPC after error@2064428585) -> (continuation of Deferred@155510905 after retry RPC after error@1029014978) -> (continuation of Deferred@1033847851 after retry RPC after error@595380311) -> (continuation of Deferred@232123233 after retry RPC after error@482284684) -> (continuation of Deferred@1531356460 after retry RPC after error@515764323) -> (continuation of Deferred@1007787574 after retry RPC after error@1483667020) -> (continuation of Deferred@921655208 after retry RPC after error@1193318398) -> (continuation of Deferred@2003849474 after retry RPC after error@1484981558) -> (continuation of Deferred@337468313 after retry RPC after error@1472307587) -> (continuation of Deferred@575410744 after retry RPC after error@1646660082) -> (continuation of Deferred@471418812 after retry RPC after error@1183723525) -> (continuation of Deferred@68237118 after retry RPC after error@1454032855)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.access$200(DefaultPromise.java:35) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise$1.run(DefaultPromise.java:503) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.10.214.190:41139
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=26548), Traces: [0ms] refreshing cache from master, [36ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.189:36301, [109ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.190:41139, [113ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.188:40593, [169ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.190:41139: Network error: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [168ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.189:36301: Network error: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [590ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.188:40593: OK, [595ms] delaying RPC due to: Service unavailable: Master config (127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [615ms] refreshing cache from master, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.189:36301, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.190:41139, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.188:40593, [625ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.189:36301: Network error: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [628ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.190:41139: Network error: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [630ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.188:40593: OK, [632ms] delaying RPC due to: Service unavailable: Master config (127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [635ms] refreshing cache from master, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.189:36301, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.190:41139, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.188:40593, [638ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.189:36301: Network error: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [640ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.190:41139: Network error: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [642ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.188:40593: OK, [643ms] delaying RPC due to: Service unavailable: Master config (127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [656ms] refreshing cache from master, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.189:36301, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.190:41139, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.188:40593, [659ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.189:36301: Network error: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [662ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.190:41139: Network error: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [664ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.188:40593: OK, [665ms] delaying RPC due to: Service unavailable: Master config (127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [695ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.189:36301, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.190:41139, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.188:40593, [699ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.189:36301: Network error: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [705ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.188:40593: OK, [706ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.190:41139: Network error: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [708ms] delaying RPC due to: Service unavailable: Master config (127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [736ms] refreshing cache from master, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.189:36301, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.190:41139, [739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.188:40593, [740ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.189:36301: Network error: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [743ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.190:41139: Network error: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [744ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.188:40593: OK, [745ms] delaying RPC due to: Service unavailable: Master config (127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [775ms] refreshing cache from master, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.189:36301, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.190:41139, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.188:40593, [779ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.189:36301: Network error: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [781ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.190:41139: Network error: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [783ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.188:40593: OK, [784ms] delaying RPC due to: Service unavailable: Master config (127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [895ms] refreshing cache from master, [895ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.189:36301, [897ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.190:41139, [898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.188:40593, [899ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.189:36301: Network error: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [901ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.190:41139: Network error: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [903ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.188:40593: OK, [904ms] delaying RPC due to: Service unavailable: Master config (127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [976ms] refreshing cache from master, [976ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.189:36301, [977ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.190:41139, [978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.188:40593, [983ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.189:36301: Network error: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [987ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.190:41139: Network error: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [989ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.188:40593: OK, [990ms] delaying RPC due to: Service unavailable: Master config (127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [1295ms] refreshing cache from master, [1296ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.189:36301, [1297ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.190:41139, [1298ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.188:40593, [1299ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.189:36301: Network error: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [1301ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.190:41139: Network error: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [1303ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.188:40593: OK, [1304ms] delaying RPC due to: Service unavailable: Master config (127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [1696ms] refreshing cache from master, [1696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.189:36301, [1698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.190:41139, [1699ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.188:40593, [1699ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.189:36301: Network error: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [1701ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.190:41139: Network error: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [1703ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.188:40593: OK, [1704ms] delaying RPC due to: Service unavailable: Master config (127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [1855ms] refreshing cache from master, [1855ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.189:36301, [1857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.190:41139, [1858ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.189:36301: Network error: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [1859ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.188:40593, [1863ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.188:40593: OK, [1864ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.190:41139: Network error: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [1865ms] delaying RPC due to: Service unavailable: Master config (127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139, [5496ms] refreshing cache from master, [5496ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.189:36301, [5497ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.214.190:41139, [5499ms] Sub RPC ConnectToMaster: received response from server master-127.10.214.189:36301: Network error: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301, [5500ms] too many traces: truncated at 100 traces, deferred=Deferred@1849646869(state=PENDING, result=null, callback=(continuation of Deferred@28832196 after retry RPC after error@344756087) -> (continuation of Deferred@1226598457 after retry RPC after error@1573863928) -> (continuation of Deferred@1674097236 after retry RPC after error@323972717) -> (continuation of Deferred@960004509 after retry RPC after error@509597635) -> (continuation of Deferred@950246478 after retry RPC after error@1350021425) -> (continuation of Deferred@759097035 after retry RPC after error@1296026976) -> (continuation of Deferred@1775925605 after retry RPC after error@80327510) -> (continuation of Deferred@1578833338 after retry RPC after error@1836877780) -> (continuation of Deferred@1473759823 after retry RPC after error@508395744) -> (continuation of Deferred@410906710 after retry RPC after error@297174698) -> (continuation of Deferred@2019210081 after retry RPC after error@174224906) -> (continuation of Deferred@2112538704 after retry RPC after error@1489801873) -> (continuation of Deferred@1973482322 after retry RPC after error@1818954186) -> (continuation of Deferred@1206445151 after retry RPC after error@2064428585) -> (continuation of Deferred@155510905 after retry RPC after error@1029014978) -> (continuation of Deferred@1033847851 after retry RPC after error@595380311) -> (continuation of Deferred@232123233 after retry RPC after error@482284684) -> (continuation of Deferred@1531356460 after retry RPC after error@515764323) -> (continuation of Deferred@1007787574 after retry RPC after error@1483667020) -> (continuation of Deferred@921655208 after retry RPC after error@1193318398) -> (continuation of Deferred@2003849474 after retry RPC after error@1484981558) -> (continuation of Deferred@337468313 after retry RPC after error@1472307587) -> (continuation of Deferred@575410744 after retry RPC after error@1646660082) -> (continuation of Deferred@471418812 after retry RPC after error@1183723525) -> (continuation of Deferred@68237118 after retry RPC after error@1454032855), errback=(continuation of Deferred@28832196 after retry RPC after error@344756087) -> (continuation of Deferred@1226598457 after retry RPC after error@1573863928) -> (continuation of Deferred@1674097236 after retry RPC after error@323972717) -> (continuation of Deferred@960004509 after retry RPC after error@509597635) -> (continuation of Deferred@950246478 after retry RPC after error@1350021425) -> (continuation of Deferred@759097035 after retry RPC after error@1296026976) -> (continuation of Deferred@1775925605 after retry RPC after error@80327510) -> (continuation of Deferred@1578833338 after retry RPC after error@1836877780) -> (continuation of Deferred@1473759823 after retry RPC after error@508395744) -> (continuation of Deferred@410906710 after retry RPC after error@297174698) -> (continuation of Deferred@2019210081 after retry RPC after error@174224906) -> (continuation of Deferred@2112538704 after retry RPC after error@1489801873) -> (continuation of Deferred@1973482322 after retry RPC after error@1818954186) -> (continuation of Deferred@1206445151 after retry RPC after error@2064428585) -> (continuation of Deferred@155510905 after retry RPC after error@1029014978) -> (continuation of Deferred@1033847851 after retry RPC after error@595380311) -> (continuation of Deferred@232123233 after retry RPC after error@482284684) -> (continuation of Deferred@1531356460 after retry RPC after error@515764323) -> (continuation of Deferred@1007787574 after retry RPC after error@1483667020) -> (continuation of Deferred@921655208 after retry RPC after error@1193318398) -> (continuation of Deferred@2003849474 after retry RPC after error@1484981558) -> (continuation of Deferred@337468313 after retry RPC after error@1472307587) -> (continuation of Deferred@575410744 after retry RPC after error@1646660082) -> (continuation of Deferred@471418812 after retry RPC after error@1183723525) -> (continuation of Deferred@68237118 after retry RPC after error@1454032855)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.10.214.189:36301,127.10.214.190:41139,127.10.214.188:40593) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.189:36301(127.10.214.189:36301): Connection refused: /127.10.214.189:36301,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.214.190:41139(127.10.214.190:41139): Connection refused: /127.10.214.190:41139
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557)
at io.netty.util.concurrent.DefaultPromise.access$200(DefaultPromise.java:35)
at io.netty.util.concurrent.DefaultPromise$1.run(DefaultPromise.java:503)
at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174)
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167)
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569)
... 5 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.10.214.190:41139
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)