20:01:36.312 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
20:01:36.326 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskBduYyb/test-tmp
20:01:36.329 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
20:01:36.333 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskBduYyb/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
20:01:36.592 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
20:01:36.758 [DEBUG - main] (MiniKuduCluster.java:195) Response:
20:01:36.759 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
20:01:36.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:36.763751 11227 mini_oidc.cc:152] Starting JWKS server
20:01:36.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:36.768296 11227 webserver.cc:460] Webserver started at http://127.0.0.1:38363/ using document root <none> and password file <none>
20:01:36.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:36.768992 11227 mini_oidc.cc:189] Starting OIDC Discovery server
20:01:36.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:36.771118 11227 webserver.cc:460] Webserver started at http://127.0.0.1:35199/ using document root <none> and password file <none>
20:01:36.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-11T20:01:36Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
20:01:36.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-11T20:01:36Z Disabled control of system clock
20:01:36.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:36.829315 11227 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskBduYyb/build/tsan/bin/kudu
20:01:36.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskBduYyb/build/tsan/bin/kudu
20:01:36.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/master-0/wal
20:01:36.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/master-0/data
20:01:36.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:01:36.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:01:36.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:01:36.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:36.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:36.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:36.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/master-0/logs
20:01:36.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/master-0/data/info.pb
20:01:36.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:36.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:01:36.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:01:36.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:01:36.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:01:36.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:36.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:36.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:36.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:01:36.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:01:36.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:01:36.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:01:36.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.246.254:34977
20:01:36.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.246.254
20:01:36.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:01:36.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.246.212:33251
20:01:36.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:36.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:36.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:36.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:01:36.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.246.254:34977,127.10.246.253:45131,127.10.246.252:36187
20:01:36.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:01:36.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38363/jwks.json with env {}
20:01:37.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:37.140942 11243 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:01:37.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:37.141642 11243 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:01:37.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:37.142313 11243 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:01:37.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:37.173563 11243 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:01:37.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:37.173868 11243 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:01:37.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:37.174167 11243 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:01:37.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:37.174458 11243 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:01:37.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:37.174782 11243 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:01:37.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:37.175007 11243 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38363/jwks.json
20:01:37.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:37.213822 11243 master_runner.cc:386] Master server non-default flags:
20:01:37.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:37.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.246.212:33251
20:01:37.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:37.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:37.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/master-0/data
20:01:37.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/master-0/wal
20:01:37.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:01:37.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.246.254:34977,127.10.246.253:45131,127.10.246.252:36187
20:01:37.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:37.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:37.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:01:37.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.246.254:34977
20:01:37.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:01:37.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:01:37.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:01:37.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38363/jwks.json
20:01:37.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:37.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:37.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/master-0/data/info.pb
20:01:37.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.246.254
20:01:37.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:01:37.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:01:37.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:37.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:01:37.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:01:37.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:37.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/master-0/logs
20:01:37.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:37.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:01:37.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:01:37.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:01:37.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:01:37.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:01:37.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:01:37.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 18:54:03 UTC on ccycloud.quasar-qqxtnc.root.comops.site
20:01:37.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4581135
20:01:37.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:01:37.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:37.215345 11243 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:01:37.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:37.217648 11243 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:01:37.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:37.238914 11250 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:38.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:37.238897 11251 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:38.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:37.238910 11253 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:38.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:37.240893 11243 server_base.cc:1034] running on GCE node
20:01:38.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:38.609033 11243 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:01:38.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:38.612344 11243 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:01:38.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:38.615861 11243 hybrid_clock.cc:648] HybridClock initialized: now 1786478498615792 us; error 83 us; skew 500 ppm
20:01:38.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:38.616730 11243 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:01:38.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:38.627147 11243 webserver.cc:460] Webserver started at http://127.10.246.254:36839/ using document root <none> and password file <none>
20:01:38.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:38.628268 11243 fs_manager.cc:359] Metadata directory not provided
20:01:38.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:38.628469 11243 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:01:38.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:38.628906 11243 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:01:38.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:38.634516 11243 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/master-0/data/instance:
20:01:38.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d471fa534e7146e58ddcd9c942bef66b"
20:01:38.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 20:01:38 on dist-test-slave-9gcw"
20:01:38.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:38.635826 11243 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/master-0/wal/instance:
20:01:38.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d471fa534e7146e58ddcd9c942bef66b"
20:01:38.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 20:01:38 on dist-test-slave-9gcw"
20:01:38.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:38.643857 11243 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.003s
20:01:38.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:38.649959 11263 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:01:38.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:38.651454 11243 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
20:01:38.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:38.651980 11243 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/master-0/data,/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/master-0/wal
20:01:38.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d471fa534e7146e58ddcd9c942bef66b"
20:01:38.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 20:01:38 on dist-test-slave-9gcw"
20:01:38.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:38.652485 11243 fs_report.cc:383] FS layout report
20:01:38.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:01:38.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/master-0/wal
20:01:38.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/master-0/wal
20:01:38.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/master-0/data/data
20:01:38.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:01:38.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:01:38.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:01:38.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:01:38.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:01:38.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:01:38.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:01:38.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:01:38.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:01:38.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:01:38.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:01:38.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:38.705997 11243 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:01:38.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:38.707834 11243 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:01:38.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:38.708392 11243 kserver.cc:163] Server-wide thread pool size limit: 3276
20:01:38.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:38.785400 11243 rpc_server.cc:307] RPC server started. Bound to: 127.10.246.254:34977
20:01:38.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:38.785486 11325 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.246.254:34977 every 8 connection(s)
20:01:38.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:38.788564 11243 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/master-0/data/info.pb
20:01:38.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:38.794823 11326 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:01:38.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:38.795260 11227 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskBduYyb/build/tsan/bin/kudu as pid 11243
20:01:38.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:38.795943 11227 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/master-0/wal/instance
20:01:38.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:38.799710 11227 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskBduYyb/build/tsan/bin/kudu
20:01:38.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskBduYyb/build/tsan/bin/kudu
20:01:38.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/master-1/wal
20:01:38.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/master-1/data
20:01:38.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:01:38.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:01:38.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:01:38.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:38.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:38.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:38.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/master-1/logs
20:01:38.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/master-1/data/info.pb
20:01:38.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:38.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:01:38.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:01:38.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:01:38.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:01:38.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:38.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:38.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:38.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:01:38.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:01:38.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:01:38.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:01:38.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.246.253:45131
20:01:38.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.246.253
20:01:38.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:01:38.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.246.212:33251
20:01:38.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:38.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:38.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:38.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:01:38.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.246.254:34977,127.10.246.253:45131,127.10.246.252:36187
20:01:38.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:01:38.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38363/jwks.json with env {}
20:01:38.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:38.810345 11326 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.246.254" port: 34977 } has no permanent_uuid. Determining permanent_uuid...
20:01:38.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:38.842133 11326 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.246.253" port: 45131 } has no permanent_uuid. Determining permanent_uuid...
20:01:38.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:38.844594 11264 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.246.253:45131: connect: Connection refused (error 111)
20:01:38.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:38.848732 11326 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.246.253:45131: Network error: Client connection negotiation failed: client connection to 127.10.246.253:45131: connect: Connection refused (error 111)
20:01:38.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:38.898355 11326 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.246.253" port: 45131 } attempt: 1
20:01:38.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:38.903039 11326 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.246.253:45131: Network error: Client connection negotiation failed: client connection to 127.10.246.253:45131: connect: Connection refused (error 111)
20:01:38.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:38.971575 11326 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.246.253" port: 45131 } attempt: 2
20:01:38.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:38.976372 11326 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.246.253:45131: Network error: Client connection negotiation failed: client connection to 127.10.246.253:45131: connect: Connection refused (error 111)
20:01:39.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:39.067873 11326 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.246.253" port: 45131 } attempt: 3
20:01:39.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:39.072105 11326 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.246.253:45131: Network error: Client connection negotiation failed: client connection to 127.10.246.253:45131: connect: Connection refused (error 111)
20:01:39.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:39.152439 11330 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:01:39.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:39.153000 11330 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:01:39.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:39.153612 11330 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:01:39.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:39.185104 11330 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:01:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:39.185420 11330 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:01:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:39.185612 11330 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:01:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:39.185802 11330 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:01:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:39.185983 11330 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:01:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:39.186111 11330 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38363/jwks.json
20:01:39.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:39.215660 11326 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.246.253" port: 45131 } attempt: 4
20:01:39.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:39.221014 11326 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.246.253:45131: Network error: Client connection negotiation failed: client connection to 127.10.246.253:45131: connect: Connection refused (error 111)
20:01:39.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:39.223209 11330 master_runner.cc:386] Master server non-default flags:
20:01:39.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:39.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.246.212:33251
20:01:39.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:39.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:39.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/master-1/data
20:01:39.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/master-1/wal
20:01:39.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:01:39.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.246.254:34977,127.10.246.253:45131,127.10.246.252:36187
20:01:39.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:39.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:39.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:01:39.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.246.253:45131
20:01:39.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:01:39.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:01:39.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:01:39.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38363/jwks.json
20:01:39.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:39.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:39.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/master-1/data/info.pb
20:01:39.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.246.253
20:01:39.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:01:39.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:01:39.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:39.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:01:39.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:01:39.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:39.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/master-1/logs
20:01:39.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:39.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:01:39.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:01:39.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:01:39.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:01:39.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:01:39.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:01:39.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 18:54:03 UTC on ccycloud.quasar-qqxtnc.root.comops.site
20:01:39.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4581135
20:01:39.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:01:39.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:39.224426 11330 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:01:39.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:39.226202 11330 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:01:39.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:39.246037 11344 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:39.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:39.520593 11326 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.246.253" port: 45131 } attempt: 5
20:01:39.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:39.526926 11326 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.246.253:45131: Network error: Client connection negotiation failed: client connection to 127.10.246.253:45131: connect: Connection refused (error 111)
20:01:40.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:40.074460 11326 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.246.253" port: 45131 } attempt: 6
20:01:40.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:40.081620 11326 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.246.253:45131: Network error: Client connection negotiation failed: client connection to 127.10.246.253:45131: connect: Connection refused (error 111)
20:01:40.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:39.246037 11347 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:40.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:39.247404 11330 server_base.cc:1034] running on GCE node
20:01:40.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:39.246141 11343 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:40.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:40.500308 11330 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:01:40.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:40.503424 11330 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:01:40.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:40.504873 11330 hybrid_clock.cc:648] HybridClock initialized: now 1786478500504830 us; error 47 us; skew 500 ppm
20:01:40.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:40.505990 11330 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:01:40.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:40.513607 11330 webserver.cc:460] Webserver started at http://127.10.246.253:45329/ using document root <none> and password file <none>
20:01:40.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:40.514882 11330 fs_manager.cc:359] Metadata directory not provided
20:01:40.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:40.515161 11330 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:01:40.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:40.515802 11330 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:01:40.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:40.521672 11330 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/master-1/data/instance:
20:01:40.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ca6c070334f34635bb008ed8fcd0f9e5"
20:01:40.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 20:01:40 on dist-test-slave-9gcw"
20:01:40.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:40.523114 11330 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/master-1/wal/instance:
20:01:40.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ca6c070334f34635bb008ed8fcd0f9e5"
20:01:40.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 20:01:40 on dist-test-slave-9gcw"
20:01:40.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:40.531448 11330 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.001s sys 0.008s
20:01:40.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:40.538231 11358 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:01:40.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:40.539469 11330 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
20:01:40.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:40.539955 11330 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/master-1/data,/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/master-1/wal
20:01:40.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ca6c070334f34635bb008ed8fcd0f9e5"
20:01:40.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 20:01:40 on dist-test-slave-9gcw"
20:01:40.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:40.540408 11330 fs_report.cc:383] FS layout report
20:01:40.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:01:40.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/master-1/wal
20:01:40.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/master-1/wal
20:01:40.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/master-1/data/data
20:01:40.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:01:40.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:01:40.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:01:40.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:01:40.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:01:40.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:01:40.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:01:40.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:01:40.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:01:40.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:01:40.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:01:40.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:40.594386 11330 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:01:40.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:40.596199 11330 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:01:40.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:40.596836 11330 kserver.cc:163] Server-wide thread pool size limit: 3276
20:01:40.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:40.678930 11330 rpc_server.cc:307] RPC server started. Bound to: 127.10.246.253:45131
20:01:40.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:40.679035 11415 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.246.253:45131 every 8 connection(s)
20:01:40.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:40.682276 11330 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/master-1/data/info.pb
20:01:40.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:40.688376 11416 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:01:40.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:40.692055 11227 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskBduYyb/build/tsan/bin/kudu as pid 11330
20:01:40.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:40.692456 11227 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/master-1/wal/instance
20:01:40.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:40.694921 11227 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskBduYyb/build/tsan/bin/kudu
20:01:40.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskBduYyb/build/tsan/bin/kudu
20:01:40.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/master-2/wal
20:01:40.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/master-2/data
20:01:40.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:01:40.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:01:40.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:01:40.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:40.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:40.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:40.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/master-2/logs
20:01:40.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/master-2/data/info.pb
20:01:40.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:40.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:01:40.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:01:40.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:01:40.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:01:40.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:40.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:40.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:40.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:01:40.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:01:40.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:01:40.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:01:40.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.246.252:36187
20:01:40.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.246.252
20:01:40.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:01:40.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.246.212:33251
20:01:40.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:40.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:40.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:40.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:01:40.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.246.254:34977,127.10.246.253:45131,127.10.246.252:36187
20:01:40.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:01:40.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38363/jwks.json with env {}
20:01:40.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:40.702508 11416 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.246.254" port: 34977 } has no permanent_uuid. Determining permanent_uuid...
20:01:40.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:40.733422 11416 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.246.253" port: 45131 } has no permanent_uuid. Determining permanent_uuid...
20:01:40.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:40.751153 11416 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.246.252" port: 36187 } has no permanent_uuid. Determining permanent_uuid...
20:01:40.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:40.753804 11361 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.246.252:36187: connect: Connection refused (error 111)
20:01:40.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:40.761508 11416 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.246.252:36187: Network error: Client connection negotiation failed: client connection to 127.10.246.252:36187: connect: Connection refused (error 111)
20:01:40.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:40.811170 11416 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.246.252" port: 36187 } attempt: 1
20:01:40.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:40.816038 11416 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.246.252:36187: Network error: Client connection negotiation failed: client connection to 127.10.246.252:36187: connect: Connection refused (error 111)
20:01:40.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:40.884570 11416 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.246.252" port: 36187 } attempt: 2
20:01:40.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:40.888849 11416 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.246.252:36187: Network error: Client connection negotiation failed: client connection to 127.10.246.252:36187: connect: Connection refused (error 111)
20:01:40.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:40.980268 11416 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.246.252" port: 36187 } attempt: 3
20:01:40.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:40.984329 11416 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.246.252:36187: Network error: Client connection negotiation failed: client connection to 127.10.246.252:36187: connect: Connection refused (error 111)
20:01:41.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:41.040690 11419 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:01:41.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:41.041407 11419 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:01:41.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:41.041906 11419 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:01:41.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:41.073820 11419 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:01:41.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:41.074122 11419 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:01:41.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:41.074313 11419 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:01:41.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:41.074493 11419 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:01:41.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:41.074688 11419 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:01:41.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:41.074822 11419 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38363/jwks.json
20:01:41.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:41.111157 11419 master_runner.cc:386] Master server non-default flags:
20:01:41.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:41.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.246.212:33251
20:01:41.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:41.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:41.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/master-2/data
20:01:41.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/master-2/wal
20:01:41.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:01:41.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.246.254:34977,127.10.246.253:45131,127.10.246.252:36187
20:01:41.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:41.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:41.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:01:41.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.246.252:36187
20:01:41.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:01:41.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:01:41.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:01:41.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38363/jwks.json
20:01:41.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:41.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:41.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/master-2/data/info.pb
20:01:41.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.246.252
20:01:41.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:01:41.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:01:41.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:41.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:01:41.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:01:41.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:41.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/master-2/logs
20:01:41.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:41.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:01:41.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:01:41.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:01:41.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:01:41.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:01:41.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:01:41.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 18:54:03 UTC on ccycloud.quasar-qqxtnc.root.comops.site
20:01:41.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4581135
20:01:41.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:01:41.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:41.112546 11419 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:01:41.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:41.114799 11419 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:01:41.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:41.127806 11416 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.246.252" port: 36187 } attempt: 4
20:01:41.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:41.132589 11416 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.246.252:36187: Network error: Client connection negotiation failed: client connection to 127.10.246.252:36187: connect: Connection refused (error 111)
20:01:41.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:41.137013 11434 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:41.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:41.142254 11326 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.246.253" port: 45131 } attempt: 7
20:01:41.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:41.155613 11326 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.246.252" port: 36187 } has no permanent_uuid. Determining permanent_uuid...
20:01:41.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:41.163620 11326 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.246.252:36187: Network error: Client connection negotiation failed: client connection to 127.10.246.252:36187: connect: Connection refused (error 111)
20:01:41.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:41.222312 11326 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.246.252" port: 36187 } attempt: 1
20:01:41.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:41.228463 11326 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.246.252:36187: Network error: Client connection negotiation failed: client connection to 127.10.246.252:36187: connect: Connection refused (error 111)
20:01:41.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:41.309983 11326 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.246.252" port: 36187 } attempt: 2
20:01:41.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:41.314977 11326 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.246.252:36187: Network error: Client connection negotiation failed: client connection to 127.10.246.252:36187: connect: Connection refused (error 111)
20:01:41.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:41.400466 11326 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.246.252" port: 36187 } attempt: 3
20:01:41.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:41.405562 11326 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.246.252:36187: Network error: Client connection negotiation failed: client connection to 127.10.246.252:36187: connect: Connection refused (error 111)
20:01:41.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:41.432029 11416 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.246.252" port: 36187 } attempt: 5
20:01:41.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:41.437537 11416 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.246.252:36187: Network error: Client connection negotiation failed: client connection to 127.10.246.252:36187: connect: Connection refused (error 111)
20:01:41.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:41.546010 11326 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.246.252" port: 36187 } attempt: 4
20:01:41.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:41.551151 11326 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.246.252:36187: Network error: Client connection negotiation failed: client connection to 127.10.246.252:36187: connect: Connection refused (error 111)
20:01:41.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:41.834587 11326 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.246.252" port: 36187 } attempt: 5
20:01:41.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:41.839818 11326 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.246.252:36187: Network error: Client connection negotiation failed: client connection to 127.10.246.252:36187: connect: Connection refused (error 111)
20:01:41.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:41.985265 11416 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.246.252" port: 36187 } attempt: 6
20:01:41.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:41.992748 11416 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.246.252:36187: Network error: Client connection negotiation failed: client connection to 127.10.246.252:36187: connect: Connection refused (error 111)
20:01:42.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:41.136993 11432 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:42.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:41.137005 11431 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:42.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:42.371196 11433 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
20:01:42.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:42.371338 11419 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:01:42.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:42.375828 11419 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:01:42.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:42.378554 11419 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:01:42.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:42.379942 11419 hybrid_clock.cc:648] HybridClock initialized: now 1786478502379917 us; error 48 us; skew 500 ppm
20:01:42.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:42.380841 11419 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:01:42.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:42.387557 11419 webserver.cc:460] Webserver started at http://127.10.246.252:40689/ using document root <none> and password file <none>
20:01:42.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:42.388576 11419 fs_manager.cc:359] Metadata directory not provided
20:01:42.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:42.388773 11419 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:01:42.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:42.389242 11419 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:01:42.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:42.392323 11326 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.246.252" port: 36187 } attempt: 6
20:01:42.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:42.395411 11419 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/master-2/data/instance:
20:01:42.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fa8431d711a449d29770a8b3140a947b"
20:01:42.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 20:01:42 on dist-test-slave-9gcw"
20:01:42.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:42.397504 11419 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/master-2/wal/instance:
20:01:42.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fa8431d711a449d29770a8b3140a947b"
20:01:42.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 20:01:42 on dist-test-slave-9gcw"
20:01:42.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:42.399621 11326 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.246.252:36187: Network error: Client connection negotiation failed: client connection to 127.10.246.252:36187: connect: Connection refused (error 111)
20:01:42.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:42.406265 11419 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.007s sys 0.001s
20:01:42.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:42.412983 11448 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:01:42.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:42.414281 11419 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
20:01:42.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:42.414579 11419 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/master-2/data,/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/master-2/wal
20:01:42.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fa8431d711a449d29770a8b3140a947b"
20:01:42.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 20:01:42 on dist-test-slave-9gcw"
20:01:42.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:42.414911 11419 fs_report.cc:383] FS layout report
20:01:42.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:01:42.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/master-2/wal
20:01:42.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/master-2/wal
20:01:42.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/master-2/data/data
20:01:42.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:01:42.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:01:42.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:01:42.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:01:42.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:01:42.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:01:42.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:01:42.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:01:42.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:01:42.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:01:42.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:01:42.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:42.451192 11419 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:01:42.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:42.452764 11419 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:01:42.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:42.453289 11419 kserver.cc:163] Server-wide thread pool size limit: 3276
20:01:42.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:42.531332 11419 rpc_server.cc:307] RPC server started. Bound to: 127.10.246.252:36187
20:01:42.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:42.531441 11512 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.246.252:36187 every 8 connection(s)
20:01:42.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:42.534500 11419 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/master-2/data/info.pb
20:01:42.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:42.540536 11227 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskBduYyb/build/tsan/bin/kudu as pid 11419
20:01:42.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:42.540848 11513 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:01:42.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:42.541074 11227 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/master-2/wal/instance
20:01:42.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:42.554986 11513 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.246.254" port: 34977 } has no permanent_uuid. Determining permanent_uuid...
20:01:42.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:42.584326 11513 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.246.253" port: 45131 } has no permanent_uuid. Determining permanent_uuid...
20:01:42.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:42.603653 11513 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.246.252" port: 36187 } has no permanent_uuid. Determining permanent_uuid...
20:01:42.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:42.623814 11513 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P fa8431d711a449d29770a8b3140a947b: Bootstrap starting.
20:01:42.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:42.631619 11513 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P fa8431d711a449d29770a8b3140a947b: Neither blocks nor log segments found. Creating new log.
20:01:42.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:42.633596 11513 log.cc:824] T 00000000000000000000000000000000 P fa8431d711a449d29770a8b3140a947b: Log is configured to *not* fsync() on all Append() calls
20:01:42.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:42.640018 11513 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P fa8431d711a449d29770a8b3140a947b: No bootstrap required, opened a new log
20:01:42.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:42.662842 11513 raft_consensus.cc:348] T 00000000000000000000000000000000 P fa8431d711a449d29770a8b3140a947b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d471fa534e7146e58ddcd9c942bef66b" member_type: VOTER last_known_addr { host: "127.10.246.254" port: 34977 } } peers { permanent_uuid: "ca6c070334f34635bb008ed8fcd0f9e5" member_type: VOTER last_known_addr { host: "127.10.246.253" port: 45131 } } peers { permanent_uuid: "fa8431d711a449d29770a8b3140a947b" member_type: VOTER last_known_addr { host: "127.10.246.252" port: 36187 } }
20:01:42.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:42.663551 11513 raft_consensus.cc:374] T 00000000000000000000000000000000 P fa8431d711a449d29770a8b3140a947b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:01:42.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:42.663843 11513 raft_consensus.cc:729] T 00000000000000000000000000000000 P fa8431d711a449d29770a8b3140a947b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fa8431d711a449d29770a8b3140a947b, State: Initialized, Role: FOLLOWER
20:01:42.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:42.664841 11513 consensus_queue.cc:260] T 00000000000000000000000000000000 P fa8431d711a449d29770a8b3140a947b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d471fa534e7146e58ddcd9c942bef66b" member_type: VOTER last_known_addr { host: "127.10.246.254" port: 34977 } } peers { permanent_uuid: "ca6c070334f34635bb008ed8fcd0f9e5" member_type: VOTER last_known_addr { host: "127.10.246.253" port: 45131 } } peers { permanent_uuid: "fa8431d711a449d29770a8b3140a947b" member_type: VOTER last_known_addr { host: "127.10.246.252" port: 36187 } }
20:01:42.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:42.667531 11525 sys_catalog.cc:455] T 00000000000000000000000000000000 P fa8431d711a449d29770a8b3140a947b [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d471fa534e7146e58ddcd9c942bef66b" member_type: VOTER last_known_addr { host: "127.10.246.254" port: 34977 } } peers { permanent_uuid: "ca6c070334f34635bb008ed8fcd0f9e5" member_type: VOTER last_known_addr { host: "127.10.246.253" port: 45131 } } peers { permanent_uuid: "fa8431d711a449d29770a8b3140a947b" member_type: VOTER last_known_addr { host: "127.10.246.252" port: 36187 } } }
20:01:42.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:42.668256 11525 sys_catalog.cc:458] T 00000000000000000000000000000000 P fa8431d711a449d29770a8b3140a947b [sys.catalog]: This master's current role is: FOLLOWER
20:01:42.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:42.669029 11513 sys_catalog.cc:564] T 00000000000000000000000000000000 P fa8431d711a449d29770a8b3140a947b [sys.catalog]: configured and running, proceeding with master startup.
20:01:42.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:42.697635 11536 catalog_manager.cc:1558] T 00000000000000000000000000000000 P fa8431d711a449d29770a8b3140a947b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:01:42.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:42.698468 11536 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:01:42.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:42.709923 11455 tablet.cc:2369] T 00000000000000000000000000000000 P fa8431d711a449d29770a8b3140a947b: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:01:42.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:42.693306 11419 master_runner.cc:186] Error getting master registration for 127.10.246.254:34977: OK, instance_id { permanent_uuid: "d471fa534e7146e58ddcd9c942bef66b" instance_seqno: 1786478498763779 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:01:42.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:42.720006 11419 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:01:43.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.053300 11416 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.246.252" port: 36187 } attempt: 7
20:01:43.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.075534 11416 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ca6c070334f34635bb008ed8fcd0f9e5: Bootstrap starting.
20:01:43.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.082500 11416 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P ca6c070334f34635bb008ed8fcd0f9e5: Neither blocks nor log segments found. Creating new log.
20:01:43.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.084724 11416 log.cc:824] T 00000000000000000000000000000000 P ca6c070334f34635bb008ed8fcd0f9e5: Log is configured to *not* fsync() on all Append() calls
20:01:43.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.090588 11416 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ca6c070334f34635bb008ed8fcd0f9e5: No bootstrap required, opened a new log
20:01:43.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.096565 11525 raft_consensus.cc:482] T 00000000000000000000000000000000 P fa8431d711a449d29770a8b3140a947b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:01:43.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.097291 11525 raft_consensus.cc:504] T 00000000000000000000000000000000 P fa8431d711a449d29770a8b3140a947b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d471fa534e7146e58ddcd9c942bef66b" member_type: VOTER last_known_addr { host: "127.10.246.254" port: 34977 } } peers { permanent_uuid: "ca6c070334f34635bb008ed8fcd0f9e5" member_type: VOTER last_known_addr { host: "127.10.246.253" port: 45131 } } peers { permanent_uuid: "fa8431d711a449d29770a8b3140a947b" member_type: VOTER last_known_addr { host: "127.10.246.252" port: 36187 } }
20:01:43.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.101135 11525 leader_election.cc:288] T 00000000000000000000000000000000 P fa8431d711a449d29770a8b3140a947b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d471fa534e7146e58ddcd9c942bef66b (127.10.246.254:34977), ca6c070334f34635bb008ed8fcd0f9e5 (127.10.246.253:45131)
20:01:43.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.123217 11416 raft_consensus.cc:348] T 00000000000000000000000000000000 P ca6c070334f34635bb008ed8fcd0f9e5 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d471fa534e7146e58ddcd9c942bef66b" member_type: VOTER last_known_addr { host: "127.10.246.254" port: 34977 } } peers { permanent_uuid: "ca6c070334f34635bb008ed8fcd0f9e5" member_type: VOTER last_known_addr { host: "127.10.246.253" port: 45131 } } peers { permanent_uuid: "fa8431d711a449d29770a8b3140a947b" member_type: VOTER last_known_addr { host: "127.10.246.252" port: 36187 } }
20:01:43.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.124236 11416 raft_consensus.cc:374] T 00000000000000000000000000000000 P ca6c070334f34635bb008ed8fcd0f9e5 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:01:43.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.124588 11416 raft_consensus.cc:729] T 00000000000000000000000000000000 P ca6c070334f34635bb008ed8fcd0f9e5 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ca6c070334f34635bb008ed8fcd0f9e5, State: Initialized, Role: FOLLOWER
20:01:43.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.125677 11416 consensus_queue.cc:260] T 00000000000000000000000000000000 P ca6c070334f34635bb008ed8fcd0f9e5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d471fa534e7146e58ddcd9c942bef66b" member_type: VOTER last_known_addr { host: "127.10.246.254" port: 34977 } } peers { permanent_uuid: "ca6c070334f34635bb008ed8fcd0f9e5" member_type: VOTER last_known_addr { host: "127.10.246.253" port: 45131 } } peers { permanent_uuid: "fa8431d711a449d29770a8b3140a947b" member_type: VOTER last_known_addr { host: "127.10.246.252" port: 36187 } }
20:01:43.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.130075 11540 sys_catalog.cc:455] T 00000000000000000000000000000000 P ca6c070334f34635bb008ed8fcd0f9e5 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d471fa534e7146e58ddcd9c942bef66b" member_type: VOTER last_known_addr { host: "127.10.246.254" port: 34977 } } peers { permanent_uuid: "ca6c070334f34635bb008ed8fcd0f9e5" member_type: VOTER last_known_addr { host: "127.10.246.253" port: 45131 } } peers { permanent_uuid: "fa8431d711a449d29770a8b3140a947b" member_type: VOTER last_known_addr { host: "127.10.246.252" port: 36187 } } }
20:01:43.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.131747 11416 sys_catalog.cc:564] T 00000000000000000000000000000000 P ca6c070334f34635bb008ed8fcd0f9e5 [sys.catalog]: configured and running, proceeding with master startup.
20:01:43.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.131867 11540 sys_catalog.cc:458] T 00000000000000000000000000000000 P ca6c070334f34635bb008ed8fcd0f9e5 [sys.catalog]: This master's current role is: FOLLOWER
20:01:43.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.131954 11389 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "fa8431d711a449d29770a8b3140a947b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ca6c070334f34635bb008ed8fcd0f9e5" is_pre_election: true
20:01:43.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.132866 11389 raft_consensus.cc:2454] T 00000000000000000000000000000000 P ca6c070334f34635bb008ed8fcd0f9e5 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate fa8431d711a449d29770a8b3140a947b in term 0.
20:01:43.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.134352 11449 leader_election.cc:302] T 00000000000000000000000000000000 P fa8431d711a449d29770a8b3140a947b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: ca6c070334f34635bb008ed8fcd0f9e5, fa8431d711a449d29770a8b3140a947b; no voters:
20:01:43.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.135385 11525 raft_consensus.cc:2781] T 00000000000000000000000000000000 P fa8431d711a449d29770a8b3140a947b [term 0 FOLLOWER]: Leader pre-election won for term 1
20:01:43.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.135740 11525 raft_consensus.cc:482] T 00000000000000000000000000000000 P fa8431d711a449d29770a8b3140a947b [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
20:01:43.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.136117 11525 raft_consensus.cc:3037] T 00000000000000000000000000000000 P fa8431d711a449d29770a8b3140a947b [term 0 FOLLOWER]: Advancing to term 1
20:01:43.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.143456 11525 raft_consensus.cc:504] T 00000000000000000000000000000000 P fa8431d711a449d29770a8b3140a947b [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d471fa534e7146e58ddcd9c942bef66b" member_type: VOTER last_known_addr { host: "127.10.246.254" port: 34977 } } peers { permanent_uuid: "ca6c070334f34635bb008ed8fcd0f9e5" member_type: VOTER last_known_addr { host: "127.10.246.253" port: 45131 } } peers { permanent_uuid: "fa8431d711a449d29770a8b3140a947b" member_type: VOTER last_known_addr { host: "127.10.246.252" port: 36187 } }
20:01:43.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.146442 11389 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "fa8431d711a449d29770a8b3140a947b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ca6c070334f34635bb008ed8fcd0f9e5"
20:01:43.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.146981 11389 raft_consensus.cc:3037] T 00000000000000000000000000000000 P ca6c070334f34635bb008ed8fcd0f9e5 [term 0 FOLLOWER]: Advancing to term 1
20:01:43.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.153646 11389 raft_consensus.cc:2454] T 00000000000000000000000000000000 P ca6c070334f34635bb008ed8fcd0f9e5 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate fa8431d711a449d29770a8b3140a947b in term 1.
20:01:43.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.154696 11449 leader_election.cc:302] T 00000000000000000000000000000000 P fa8431d711a449d29770a8b3140a947b [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: ca6c070334f34635bb008ed8fcd0f9e5, fa8431d711a449d29770a8b3140a947b; no voters:
20:01:43.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.160565 11525 leader_election.cc:288] T 00000000000000000000000000000000 P fa8431d711a449d29770a8b3140a947b [CANDIDATE]: Term 1 election: Requested vote from peers d471fa534e7146e58ddcd9c942bef66b (127.10.246.254:34977), ca6c070334f34635bb008ed8fcd0f9e5 (127.10.246.253:45131)
20:01:43.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.161069 11525 raft_consensus.cc:2781] T 00000000000000000000000000000000 P fa8431d711a449d29770a8b3140a947b [term 1 FOLLOWER]: Leader election won for term 1
20:01:43.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.163939 11525 raft_consensus.cc:686] T 00000000000000000000000000000000 P fa8431d711a449d29770a8b3140a947b [term 1 LEADER]: Becoming Leader. State: Replica: fa8431d711a449d29770a8b3140a947b, State: Running, Role: LEADER
20:01:43.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.166095 11525 consensus_queue.cc:237] T 00000000000000000000000000000000 P fa8431d711a449d29770a8b3140a947b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d471fa534e7146e58ddcd9c942bef66b" member_type: VOTER last_known_addr { host: "127.10.246.254" port: 34977 } } peers { permanent_uuid: "ca6c070334f34635bb008ed8fcd0f9e5" member_type: VOTER last_known_addr { host: "127.10.246.253" port: 45131 } } peers { permanent_uuid: "fa8431d711a449d29770a8b3140a947b" member_type: VOTER last_known_addr { host: "127.10.246.252" port: 36187 } }
20:01:43.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.193610 11544 sys_catalog.cc:455] T 00000000000000000000000000000000 P fa8431d711a449d29770a8b3140a947b [sys.catalog]: SysCatalogTable state changed. Reason: New leader fa8431d711a449d29770a8b3140a947b. Latest consensus state: current_term: 1 leader_uuid: "fa8431d711a449d29770a8b3140a947b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d471fa534e7146e58ddcd9c942bef66b" member_type: VOTER last_known_addr { host: "127.10.246.254" port: 34977 } } peers { permanent_uuid: "ca6c070334f34635bb008ed8fcd0f9e5" member_type: VOTER last_known_addr { host: "127.10.246.253" port: 45131 } } peers { permanent_uuid: "fa8431d711a449d29770a8b3140a947b" member_type: VOTER last_known_addr { host: "127.10.246.252" port: 36187 } } }
20:01:43.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.201706 11544 sys_catalog.cc:458] T 00000000000000000000000000000000 P fa8431d711a449d29770a8b3140a947b [sys.catalog]: This master's current role is: LEADER
20:01:43.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.212569 11558 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
20:01:43.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:43.223131 11556 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ca6c070334f34635bb008ed8fcd0f9e5: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:01:43.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:43.223913 11556 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:01:43.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.237572 11558 catalog_manager.cc:1484] Initializing Kudu cluster ID...
20:01:43.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.206153 11330 master_runner.cc:186] Error getting master registration for 127.10.246.254:34977: OK, instance_id { permanent_uuid: "d471fa534e7146e58ddcd9c942bef66b" instance_seqno: 1786478498763779 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:01:43.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.242372 11330 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:01:43.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.267796 11389 raft_consensus.cc:1261] T 00000000000000000000000000000000 P ca6c070334f34635bb008ed8fcd0f9e5 [term 1 FOLLOWER]: Refusing update from remote peer fa8431d711a449d29770a8b3140a947b: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:01:43.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.269695 11525 consensus_queue.cc:1035] T 00000000000000000000000000000000 P fa8431d711a449d29770a8b3140a947b [LEADER]: Connected to new peer: Peer: permanent_uuid: "ca6c070334f34635bb008ed8fcd0f9e5" member_type: VOTER last_known_addr { host: "127.10.246.253" port: 45131 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
20:01:43.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.311513 11540 sys_catalog.cc:455] T 00000000000000000000000000000000 P ca6c070334f34635bb008ed8fcd0f9e5 [sys.catalog]: SysCatalogTable state changed. Reason: New leader fa8431d711a449d29770a8b3140a947b. Latest consensus state: current_term: 1 leader_uuid: "fa8431d711a449d29770a8b3140a947b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d471fa534e7146e58ddcd9c942bef66b" member_type: VOTER last_known_addr { host: "127.10.246.254" port: 34977 } } peers { permanent_uuid: "ca6c070334f34635bb008ed8fcd0f9e5" member_type: VOTER last_known_addr { host: "127.10.246.253" port: 45131 } } peers { permanent_uuid: "fa8431d711a449d29770a8b3140a947b" member_type: VOTER last_known_addr { host: "127.10.246.252" port: 36187 } } }
20:01:43.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.312260 11540 sys_catalog.cc:458] T 00000000000000000000000000000000 P ca6c070334f34635bb008ed8fcd0f9e5 [sys.catalog]: This master's current role is: FOLLOWER
20:01:43.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.319424 11544 sys_catalog.cc:455] T 00000000000000000000000000000000 P fa8431d711a449d29770a8b3140a947b [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "fa8431d711a449d29770a8b3140a947b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d471fa534e7146e58ddcd9c942bef66b" member_type: VOTER last_known_addr { host: "127.10.246.254" port: 34977 } } peers { permanent_uuid: "ca6c070334f34635bb008ed8fcd0f9e5" member_type: VOTER last_known_addr { host: "127.10.246.253" port: 45131 } } peers { permanent_uuid: "fa8431d711a449d29770a8b3140a947b" member_type: VOTER last_known_addr { host: "127.10.246.252" port: 36187 } } }
20:01:43.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.320495 11544 sys_catalog.cc:458] T 00000000000000000000000000000000 P fa8431d711a449d29770a8b3140a947b [sys.catalog]: This master's current role is: LEADER
20:01:43.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.321331 11558 catalog_manager.cc:1347] Generated new cluster ID: c4d1d674ec4b4bcbb039bbcc65a78da8
20:01:43.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.321640 11558 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
20:01:43.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.328254 11540 sys_catalog.cc:455] T 00000000000000000000000000000000 P ca6c070334f34635bb008ed8fcd0f9e5 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "fa8431d711a449d29770a8b3140a947b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d471fa534e7146e58ddcd9c942bef66b" member_type: VOTER last_known_addr { host: "127.10.246.254" port: 34977 } } peers { permanent_uuid: "ca6c070334f34635bb008ed8fcd0f9e5" member_type: VOTER last_known_addr { host: "127.10.246.253" port: 45131 } } peers { permanent_uuid: "fa8431d711a449d29770a8b3140a947b" member_type: VOTER last_known_addr { host: "127.10.246.252" port: 36187 } } }
20:01:43.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.329871 11540 sys_catalog.cc:458] T 00000000000000000000000000000000 P ca6c070334f34635bb008ed8fcd0f9e5 [sys.catalog]: This master's current role is: FOLLOWER
20:01:43.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.391139 11558 catalog_manager.cc:1370] Generated new certificate authority record
20:01:43.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.394464 11558 catalog_manager.cc:1504] Loading token signing keys...
20:01:43.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.430627 11558 catalog_manager.cc:5868] T 00000000000000000000000000000000 P fa8431d711a449d29770a8b3140a947b: Generated new TSK 0
20:01:43.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.433048 11326 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.246.252" port: 36187 } attempt: 7
20:01:43.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.435704 11558 catalog_manager.cc:1514] Initializing in-progress tserver states...
20:01:43.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.478048 11326 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d471fa534e7146e58ddcd9c942bef66b: Bootstrap starting.
20:01:43.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.485889 11326 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P d471fa534e7146e58ddcd9c942bef66b: Neither blocks nor log segments found. Creating new log.
20:01:43.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.488745 11326 log.cc:824] T 00000000000000000000000000000000 P d471fa534e7146e58ddcd9c942bef66b: Log is configured to *not* fsync() on all Append() calls
20:01:43.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.500645 11326 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d471fa534e7146e58ddcd9c942bef66b: No bootstrap required, opened a new log
20:01:43.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.507155 11227 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskBduYyb/build/tsan/bin/kudu
20:01:43.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskBduYyb/build/tsan/bin/kudu
20:01:43.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/ts-0/wal
20:01:43.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/ts-0/data
20:01:43.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:01:43.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:01:43.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:01:43.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:43.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:43.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:43.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/ts-0/logs
20:01:43.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/ts-0/data/info.pb
20:01:43.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:43.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:01:43.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:01:43.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:01:43.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:01:43.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:43.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:43.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:43.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:01:43.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:01:43.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.246.193:0
20:01:43.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.246.193
20:01:43.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.246.193
20:01:43.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:01:43.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.246.254:34977,127.10.246.253:45131,127.10.246.252:36187
20:01:43.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.246.212:33251
20:01:43.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:43.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:43.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:01:43.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.532949 11326 raft_consensus.cc:348] T 00000000000000000000000000000000 P d471fa534e7146e58ddcd9c942bef66b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d471fa534e7146e58ddcd9c942bef66b" member_type: VOTER last_known_addr { host: "127.10.246.254" port: 34977 } } peers { permanent_uuid: "ca6c070334f34635bb008ed8fcd0f9e5" member_type: VOTER last_known_addr { host: "127.10.246.253" port: 45131 } } peers { permanent_uuid: "fa8431d711a449d29770a8b3140a947b" member_type: VOTER last_known_addr { host: "127.10.246.252" port: 36187 } }
20:01:43.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.534415 11326 raft_consensus.cc:374] T 00000000000000000000000000000000 P d471fa534e7146e58ddcd9c942bef66b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:01:43.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.534914 11326 raft_consensus.cc:729] T 00000000000000000000000000000000 P d471fa534e7146e58ddcd9c942bef66b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d471fa534e7146e58ddcd9c942bef66b, State: Initialized, Role: FOLLOWER
20:01:43.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.536382 11326 consensus_queue.cc:260] T 00000000000000000000000000000000 P d471fa534e7146e58ddcd9c942bef66b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d471fa534e7146e58ddcd9c942bef66b" member_type: VOTER last_known_addr { host: "127.10.246.254" port: 34977 } } peers { permanent_uuid: "ca6c070334f34635bb008ed8fcd0f9e5" member_type: VOTER last_known_addr { host: "127.10.246.253" port: 45131 } } peers { permanent_uuid: "fa8431d711a449d29770a8b3140a947b" member_type: VOTER last_known_addr { host: "127.10.246.252" port: 36187 } }
20:01:43.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.544842 11568 sys_catalog.cc:455] T 00000000000000000000000000000000 P d471fa534e7146e58ddcd9c942bef66b [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d471fa534e7146e58ddcd9c942bef66b" member_type: VOTER last_known_addr { host: "127.10.246.254" port: 34977 } } peers { permanent_uuid: "ca6c070334f34635bb008ed8fcd0f9e5" member_type: VOTER last_known_addr { host: "127.10.246.253" port: 45131 } } peers { permanent_uuid: "fa8431d711a449d29770a8b3140a947b" member_type: VOTER last_known_addr { host: "127.10.246.252" port: 36187 } } }
20:01:43.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.550086 11568 sys_catalog.cc:458] T 00000000000000000000000000000000 P d471fa534e7146e58ddcd9c942bef66b [sys.catalog]: This master's current role is: FOLLOWER
20:01:43.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.551213 11295 raft_consensus.cc:3037] T 00000000000000000000000000000000 P d471fa534e7146e58ddcd9c942bef66b [term 0 FOLLOWER]: Advancing to term 1
20:01:43.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.553462 11297 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "fa8431d711a449d29770a8b3140a947b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d471fa534e7146e58ddcd9c942bef66b" is_pre_election: true
20:01:43.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.549933 11296 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "fa8431d711a449d29770a8b3140a947b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d471fa534e7146e58ddcd9c942bef66b"
20:01:43.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.549703 11326 sys_catalog.cc:564] T 00000000000000000000000000000000 P d471fa534e7146e58ddcd9c942bef66b [sys.catalog]: configured and running, proceeding with master startup.
20:01:43.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.560528 11295 raft_consensus.cc:1261] T 00000000000000000000000000000000 P d471fa534e7146e58ddcd9c942bef66b [term 1 FOLLOWER]: Refusing update from remote peer fa8431d711a449d29770a8b3140a947b: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:01:43.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.562573 11525 consensus_queue.cc:1035] T 00000000000000000000000000000000 P fa8431d711a449d29770a8b3140a947b [LEADER]: Connected to new peer: Peer: permanent_uuid: "d471fa534e7146e58ddcd9c942bef66b" member_type: VOTER last_known_addr { host: "127.10.246.254" port: 34977 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
20:01:43.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.676716 11575 sys_catalog.cc:455] T 00000000000000000000000000000000 P d471fa534e7146e58ddcd9c942bef66b [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "fa8431d711a449d29770a8b3140a947b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d471fa534e7146e58ddcd9c942bef66b" member_type: VOTER last_known_addr { host: "127.10.246.254" port: 34977 } } peers { permanent_uuid: "ca6c070334f34635bb008ed8fcd0f9e5" member_type: VOTER last_known_addr { host: "127.10.246.253" port: 45131 } } peers { permanent_uuid: "fa8431d711a449d29770a8b3140a947b" member_type: VOTER last_known_addr { host: "127.10.246.252" port: 36187 } } }
20:01:43.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.677786 11575 sys_catalog.cc:458] T 00000000000000000000000000000000 P d471fa534e7146e58ddcd9c942bef66b [sys.catalog]: This master's current role is: FOLLOWER
20:01:43.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.678583 11568 sys_catalog.cc:455] T 00000000000000000000000000000000 P d471fa534e7146e58ddcd9c942bef66b [sys.catalog]: SysCatalogTable state changed. Reason: New leader fa8431d711a449d29770a8b3140a947b. Latest consensus state: current_term: 1 leader_uuid: "fa8431d711a449d29770a8b3140a947b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d471fa534e7146e58ddcd9c942bef66b" member_type: VOTER last_known_addr { host: "127.10.246.254" port: 34977 } } peers { permanent_uuid: "ca6c070334f34635bb008ed8fcd0f9e5" member_type: VOTER last_known_addr { host: "127.10.246.253" port: 45131 } } peers { permanent_uuid: "fa8431d711a449d29770a8b3140a947b" member_type: VOTER last_known_addr { host: "127.10.246.252" port: 36187 } } }
20:01:43.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.679468 11568 sys_catalog.cc:458] T 00000000000000000000000000000000 P d471fa534e7146e58ddcd9c942bef66b [sys.catalog]: This master's current role is: FOLLOWER
20:01:43.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.734396 11586 catalog_manager.cc:1259] Loaded cluster ID: c4d1d674ec4b4bcbb039bbcc65a78da8
20:01:43.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.735046 11586 catalog_manager.cc:1552] T 00000000000000000000000000000000 P d471fa534e7146e58ddcd9c942bef66b: loading cluster ID for follower catalog manager: success
20:01:43.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.743947 11586 catalog_manager.cc:1574] T 00000000000000000000000000000000 P d471fa534e7146e58ddcd9c942bef66b: acquiring CA information for follower catalog manager: success
20:01:43.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:43.749975 11586 catalog_manager.cc:1602] T 00000000000000000000000000000000 P d471fa534e7146e58ddcd9c942bef66b: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:01:44.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:44.105643 11525 sys_catalog.cc:455] T 00000000000000000000000000000000 P fa8431d711a449d29770a8b3140a947b [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "fa8431d711a449d29770a8b3140a947b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d471fa534e7146e58ddcd9c942bef66b" member_type: VOTER last_known_addr { host: "127.10.246.254" port: 34977 } } peers { permanent_uuid: "ca6c070334f34635bb008ed8fcd0f9e5" member_type: VOTER last_known_addr { host: "127.10.246.253" port: 45131 } } peers { permanent_uuid: "fa8431d711a449d29770a8b3140a947b" member_type: VOTER last_known_addr { host: "127.10.246.252" port: 36187 } } }
20:01:44.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:44.106833 11525 sys_catalog.cc:458] T 00000000000000000000000000000000 P fa8431d711a449d29770a8b3140a947b [sys.catalog]: This master's current role is: LEADER
20:01:44.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:44.140375 11567 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:01:44.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:44.141181 11567 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:01:44.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:44.142122 11567 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:01:44.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:44.202951 11567 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:01:44.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:44.204399 11567 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.246.193
20:01:44.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:44.230150 11556 catalog_manager.cc:1259] Loaded cluster ID: c4d1d674ec4b4bcbb039bbcc65a78da8
20:01:44.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:44.231383 11556 catalog_manager.cc:1552] T 00000000000000000000000000000000 P ca6c070334f34635bb008ed8fcd0f9e5: loading cluster ID for follower catalog manager: success
20:01:44.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:44.251083 11556 catalog_manager.cc:1574] T 00000000000000000000000000000000 P ca6c070334f34635bb008ed8fcd0f9e5: acquiring CA information for follower catalog manager: success
20:01:44.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:44.258671 11556 catalog_manager.cc:1602] T 00000000000000000000000000000000 P ca6c070334f34635bb008ed8fcd0f9e5: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:01:44.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:44.295753 11567 tablet_server_runner.cc:78] Tablet server non-default flags:
20:01:44.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:44.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.246.212:33251
20:01:44.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:44.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:44.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/ts-0/data
20:01:44.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/ts-0/wal
20:01:44.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:44.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:44.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.246.193:0
20:01:44.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:01:44.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:44.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:44.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/ts-0/data/info.pb
20:01:44.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.246.193
20:01:44.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:01:44.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.246.254:34977,127.10.246.253:45131,127.10.246.252:36187
20:01:44.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:01:44.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:44.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:01:44.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:01:44.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:44.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.246.193
20:01:44.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/ts-0/logs
20:01:44.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:44.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:01:44.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:01:44.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:01:44.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:01:44.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:01:44.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:01:44.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 18:54:03 UTC on ccycloud.quasar-qqxtnc.root.comops.site
20:01:44.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4581135
20:01:44.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:01:44.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:44.299244 11567 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:01:44.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:44.301904 11567 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:01:44.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:44.349175 11599 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:45.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:44.358650 11603 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:45.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:45.839242 11600 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
20:01:45.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:45.839471 11567 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:01:45.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:44.357045 11598 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:45.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:45.847090 11567 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:01:45.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:45.857071 11567 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:01:45.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:45.858779 11567 hybrid_clock.cc:648] HybridClock initialized: now 1786478505858746 us; error 56 us; skew 500 ppm
20:01:45.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:45.860004 11567 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:01:45.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:45.886703 11567 webserver.cc:460] Webserver started at http://127.10.246.193:41887/ using document root <none> and password file <none>
20:01:45.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:45.888141 11567 fs_manager.cc:359] Metadata directory not provided
20:01:45.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:45.888453 11567 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:01:45.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:45.889120 11567 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:01:45.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:45.897033 11567 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/ts-0/data/instance:
20:01:45.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e89e04f07c7b4656b4fe6eaf256fe473"
20:01:45.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 20:01:45 on dist-test-slave-9gcw"
20:01:45.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:45.898725 11567 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/ts-0/wal/instance:
20:01:45.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e89e04f07c7b4656b4fe6eaf256fe473"
20:01:45.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 20:01:45 on dist-test-slave-9gcw"
20:01:45.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:45.914951 11567 fs_manager.cc:691] Time spent creating directory manager: real 0.015s user 0.014s sys 0.001s
20:01:45.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:45.929188 11610 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:01:45.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:45.930629 11567 fs_manager.cc:725] Time spent opening block manager: real 0.010s user 0.010s sys 0.001s
20:01:45.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:45.931074 11567 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/ts-0/data,/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/ts-0/wal
20:01:45.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e89e04f07c7b4656b4fe6eaf256fe473"
20:01:45.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 20:01:45 on dist-test-slave-9gcw"
20:01:45.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:45.931592 11567 fs_report.cc:383] FS layout report
20:01:45.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:01:45.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/ts-0/wal
20:01:45.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/ts-0/wal
20:01:45.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/ts-0/data/data
20:01:45.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:01:45.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:01:45.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:01:45.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:01:45.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:01:45.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:01:45.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:01:45.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:01:45.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:01:45.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:01:45.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:01:46.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:46.000629 11567 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:01:46.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:46.002712 11567 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:01:46.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:46.003377 11567 kserver.cc:163] Server-wide thread pool size limit: 3276
20:01:46.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:46.006908 11567 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
20:01:46.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:46.012244 11567 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:01:46.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:46.012491 11567 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:01:46.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:46.012816 11567 ts_tablet_manager.cc:610] Registered 0 tablets
20:01:46.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:46.013011 11567 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:01:46.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:46.257833 11567 rpc_server.cc:307] RPC server started. Bound to: 127.10.246.193:40381
20:01:46.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:46.257963 11743 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.246.193:40381 every 8 connection(s)
20:01:46.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:46.262079 11567 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/ts-0/data/info.pb
20:01:46.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:46.263515 11227 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskBduYyb/build/tsan/bin/kudu as pid 11567
20:01:46.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:46.264087 11227 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/ts-0/wal/instance
20:01:46.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:46.287153 11227 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskBduYyb/build/tsan/bin/kudu
20:01:46.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskBduYyb/build/tsan/bin/kudu
20:01:46.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/ts-1/wal
20:01:46.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/ts-1/data
20:01:46.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:01:46.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:01:46.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:01:46.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:46.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:46.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:46.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/ts-1/logs
20:01:46.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/ts-1/data/info.pb
20:01:46.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:46.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:01:46.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:01:46.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:01:46.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:01:46.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:46.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:46.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:46.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:01:46.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:01:46.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.246.194:0
20:01:46.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.246.194
20:01:46.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.246.194
20:01:46.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:01:46.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.246.254:34977,127.10.246.253:45131,127.10.246.252:36187
20:01:46.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.246.212:33251
20:01:46.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:46.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:46.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:01:46.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:46.328269 11744 heartbeater.cc:347] Connected to a master server at 127.10.246.253:45131
20:01:46.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:46.328855 11744 heartbeater.cc:464] Registering TS with master...
20:01:46.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:46.330480 11744 heartbeater.cc:511] Master 127.10.246.253:45131 requested a full tablet report, sending...
20:01:46.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:46.336707 11745 heartbeater.cc:347] Connected to a master server at 127.10.246.252:36187
20:01:46.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:46.337080 11745 heartbeater.cc:464] Registering TS with master...
20:01:46.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:46.338111 11378 ts_manager.cc:194] Registered new tserver with Master: e89e04f07c7b4656b4fe6eaf256fe473 (127.10.246.193:40381)
20:01:46.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:46.338217 11745 heartbeater.cc:511] Master 127.10.246.252:36187 requested a full tablet report, sending...
20:01:46.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:46.346230 11469 ts_manager.cc:194] Registered new tserver with Master: e89e04f07c7b4656b4fe6eaf256fe473 (127.10.246.193:40381)
20:01:46.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:46.349479 11469 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.246.193:35823
20:01:46.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:46.363205 11748 heartbeater.cc:347] Connected to a master server at 127.10.246.254:34977
20:01:46.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:46.363610 11748 heartbeater.cc:464] Registering TS with master...
20:01:46.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:46.364403 11748 heartbeater.cc:511] Master 127.10.246.254:34977 requested a full tablet report, sending...
20:01:46.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:46.367859 11284 ts_manager.cc:194] Registered new tserver with Master: e89e04f07c7b4656b4fe6eaf256fe473 (127.10.246.193:40381)
20:01:46.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:46.776399 11755 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:01:46.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:46.777031 11755 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:01:46.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:46.777814 11755 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:01:46.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:46.829720 11755 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:01:46.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:46.830821 11755 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.246.194
20:01:46.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:46.868871 11755 tablet_server_runner.cc:78] Tablet server non-default flags:
20:01:46.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:46.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.246.212:33251
20:01:46.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:46.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:46.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/ts-1/data
20:01:46.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/ts-1/wal
20:01:46.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:46.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:46.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.246.194:0
20:01:46.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:01:46.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:46.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:46.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/ts-1/data/info.pb
20:01:46.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.246.194
20:01:46.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:01:46.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.246.254:34977,127.10.246.253:45131,127.10.246.252:36187
20:01:46.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:01:46.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:46.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:01:46.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:01:46.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:46.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.246.194
20:01:46.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/ts-1/logs
20:01:46.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:46.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:01:46.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:01:46.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:01:46.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:01:46.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:01:46.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:01:46.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 18:54:03 UTC on ccycloud.quasar-qqxtnc.root.comops.site
20:01:46.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4581135
20:01:46.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:01:46.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:46.870570 11755 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:01:46.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:46.872694 11755 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:01:46.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:46.893440 11763 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:47.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:47.355021 11745 heartbeater.cc:503] Master 127.10.246.252:36187 was elected leader, sending a full tablet report...
20:01:48.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:46.893831 11764 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:48.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:46.895686 11755 server_base.cc:1034] running on GCE node
20:01:48.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:46.893715 11766 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:48.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:48.218855 11755 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:01:48.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:48.221342 11755 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:01:48.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:48.222807 11755 hybrid_clock.cc:648] HybridClock initialized: now 1786478508222744 us; error 46 us; skew 500 ppm
20:01:48.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:48.223740 11755 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:01:48.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:48.231344 11755 webserver.cc:460] Webserver started at http://127.10.246.194:35437/ using document root <none> and password file <none>
20:01:48.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:48.232448 11755 fs_manager.cc:359] Metadata directory not provided
20:01:48.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:48.232784 11755 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:01:48.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:48.233412 11755 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:01:48.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:48.239151 11755 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/ts-1/data/instance:
20:01:48.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ebea53670de247d6a8b24c1e35e56a88"
20:01:48.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 20:01:48 on dist-test-slave-9gcw"
20:01:48.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:48.240458 11755 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/ts-1/wal/instance:
20:01:48.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ebea53670de247d6a8b24c1e35e56a88"
20:01:48.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 20:01:48 on dist-test-slave-9gcw"
20:01:48.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:48.250782 11755 fs_manager.cc:691] Time spent creating directory manager: real 0.010s user 0.005s sys 0.003s
20:01:48.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:48.259100 11774 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:01:48.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:48.260408 11755 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.000s sys 0.005s
20:01:48.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:48.260730 11755 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/ts-1/data,/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/ts-1/wal
20:01:48.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ebea53670de247d6a8b24c1e35e56a88"
20:01:48.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 20:01:48 on dist-test-slave-9gcw"
20:01:48.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:48.261044 11755 fs_report.cc:383] FS layout report
20:01:48.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:01:48.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/ts-1/wal
20:01:48.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/ts-1/wal
20:01:48.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/ts-1/data/data
20:01:48.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:01:48.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:01:48.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:01:48.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:01:48.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:01:48.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:01:48.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:01:48.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:01:48.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:01:48.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:01:48.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:01:48.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:48.313097 11755 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:01:48.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:48.314739 11755 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:01:48.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:48.315166 11755 kserver.cc:163] Server-wide thread pool size limit: 3276
20:01:48.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:48.317811 11755 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
20:01:48.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:48.322182 11755 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:01:48.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:48.322396 11755 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:01:48.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:48.322602 11755 ts_tablet_manager.cc:610] Registered 0 tablets
20:01:48.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:48.322741 11755 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:01:48.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:48.479038 11755 rpc_server.cc:307] RPC server started. Bound to: 127.10.246.194:36153
20:01:48.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:48.479228 11912 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.246.194:36153 every 8 connection(s)
20:01:48.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:48.482265 11755 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/ts-1/data/info.pb
20:01:48.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:48.491640 11227 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskBduYyb/build/tsan/bin/kudu as pid 11755
20:01:48.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:48.492520 11227 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/ts-1/wal/instance
20:01:48.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:48.515930 11227 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskBduYyb/build/tsan/bin/kudu
20:01:48.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskBduYyb/build/tsan/bin/kudu
20:01:48.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/ts-2/wal
20:01:48.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/ts-2/data
20:01:48.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:01:48.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:01:48.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:01:48.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:48.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:48.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:48.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/ts-2/logs
20:01:48.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/ts-2/data/info.pb
20:01:48.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:48.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:01:48.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:01:48.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:01:48.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:01:48.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:48.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:48.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:48.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:01:48.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:01:48.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.246.195:0
20:01:48.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.246.195
20:01:48.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.246.195
20:01:48.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:01:48.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.246.254:34977,127.10.246.253:45131,127.10.246.252:36187
20:01:48.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.246.212:33251
20:01:48.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:48.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:48.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:01:48.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:48.538940 11916 heartbeater.cc:347] Connected to a master server at 127.10.246.254:34977
20:01:48.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:48.539458 11916 heartbeater.cc:464] Registering TS with master...
20:01:48.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:48.540771 11916 heartbeater.cc:511] Master 127.10.246.254:34977 requested a full tablet report, sending...
20:01:48.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:48.544278 11284 ts_manager.cc:194] Registered new tserver with Master: ebea53670de247d6a8b24c1e35e56a88 (127.10.246.194:36153)
20:01:48.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:48.548152 11915 heartbeater.cc:347] Connected to a master server at 127.10.246.252:36187
20:01:48.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:48.548522 11915 heartbeater.cc:464] Registering TS with master...
20:01:48.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:48.549520 11915 heartbeater.cc:511] Master 127.10.246.252:36187 requested a full tablet report, sending...
20:01:48.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:48.549933 11913 heartbeater.cc:347] Connected to a master server at 127.10.246.253:45131
20:01:48.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:48.550288 11913 heartbeater.cc:464] Registering TS with master...
20:01:48.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:48.551086 11913 heartbeater.cc:511] Master 127.10.246.253:45131 requested a full tablet report, sending...
20:01:48.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:48.551954 11469 ts_manager.cc:194] Registered new tserver with Master: ebea53670de247d6a8b24c1e35e56a88 (127.10.246.194:36153)
20:01:48.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:48.553984 11378 ts_manager.cc:194] Registered new tserver with Master: ebea53670de247d6a8b24c1e35e56a88 (127.10.246.194:36153)
20:01:48.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:48.554212 11469 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.246.194:41291
20:01:48.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:48.862854 11927 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:01:48.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:48.863451 11927 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:01:48.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:48.864169 11927 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:01:48.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:48.899585 11927 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:01:48.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:48.900411 11927 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.246.195
20:01:48.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:48.939898 11927 tablet_server_runner.cc:78] Tablet server non-default flags:
20:01:48.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:48.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.246.212:33251
20:01:48.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:48.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:48.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/ts-2/data
20:01:48.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/ts-2/wal
20:01:48.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:48.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:48.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.246.195:0
20:01:48.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:01:48.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:48.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:48.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/ts-2/data/info.pb
20:01:48.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.246.195
20:01:48.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:01:48.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.246.254:34977,127.10.246.253:45131,127.10.246.252:36187
20:01:48.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:01:48.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:48.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:01:48.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:01:48.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:48.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.246.195
20:01:48.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/ts-2/logs
20:01:48.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:48.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:01:48.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:01:48.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:01:48.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:01:48.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:01:48.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:01:48.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 18:54:03 UTC on ccycloud.quasar-qqxtnc.root.comops.site
20:01:48.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4581135
20:01:48.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:01:48.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:48.941251 11927 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:01:48.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:48.942873 11927 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:01:48.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:48.962750 11936 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:49.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:49.558957 11915 heartbeater.cc:503] Master 127.10.246.252:36187 was elected leader, sending a full tablet report...
20:01:50.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:48.962791 11939 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:50.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:48.964063 11937 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:50.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:48.966080 11927 server_base.cc:1034] running on GCE node
20:01:50.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:50.298321 11927 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:01:50.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:50.300693 11927 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:01:50.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:50.302088 11927 hybrid_clock.cc:648] HybridClock initialized: now 1786478510302050 us; error 44 us; skew 500 ppm
20:01:50.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:50.303170 11927 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:01:50.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:50.310334 11927 webserver.cc:460] Webserver started at http://127.10.246.195:37587/ using document root <none> and password file <none>
20:01:50.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:50.311259 11927 fs_manager.cc:359] Metadata directory not provided
20:01:50.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:50.311445 11927 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:01:50.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:50.311839 11927 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:01:50.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:50.316955 11927 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/ts-2/data/instance:
20:01:50.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "762edfeaedf7488b81ac50d1e39edaa1"
20:01:50.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 20:01:50 on dist-test-slave-9gcw"
20:01:50.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:50.318200 11927 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/ts-2/wal/instance:
20:01:50.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "762edfeaedf7488b81ac50d1e39edaa1"
20:01:50.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 20:01:50 on dist-test-slave-9gcw"
20:01:50.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:50.325745 11927 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
20:01:50.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:50.331509 11947 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:01:50.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:50.332835 11927 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
20:01:50.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:50.333320 11927 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/ts-2/data,/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/ts-2/wal
20:01:50.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "762edfeaedf7488b81ac50d1e39edaa1"
20:01:50.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 20:01:50 on dist-test-slave-9gcw"
20:01:50.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:50.333711 11927 fs_report.cc:383] FS layout report
20:01:50.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:01:50.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/ts-2/wal
20:01:50.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/ts-2/wal
20:01:50.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/ts-2/data/data
20:01:50.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:01:50.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:01:50.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:01:50.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:01:50.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:01:50.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:01:50.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:01:50.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:01:50.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:01:50.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:01:50.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:01:50.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:50.373733 11927 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:01:50.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:50.375187 11927 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:01:50.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:50.375670 11927 kserver.cc:163] Server-wide thread pool size limit: 3276
20:01:50.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:50.378468 11927 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
20:01:50.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:50.382910 11927 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:01:50.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:50.383111 11927 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:01:50.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:50.383315 11927 ts_tablet_manager.cc:610] Registered 0 tablets
20:01:50.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:50.383445 11927 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:01:50.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:50.539017 11927 rpc_server.cc:307] RPC server started. Bound to: 127.10.246.195:45981
20:01:50.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:50.539171 12085 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.246.195:45981 every 8 connection(s)
20:01:50.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:50.542892 11927 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/ts-2/data/info.pb
20:01:50.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:50.551273 11227 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskBduYyb/build/tsan/bin/kudu as pid 11927
20:01:50.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:50.551739 11227 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster236395863803759496/ts-2/wal/instance
20:01:50.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:50.603385 12087 heartbeater.cc:347] Connected to a master server at 127.10.246.252:36187
20:01:50.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:50.603929 12087 heartbeater.cc:464] Registering TS with master...
20:01:50.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:50.604413 12086 heartbeater.cc:347] Connected to a master server at 127.10.246.253:45131
20:01:50.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:50.604748 12086 heartbeater.cc:464] Registering TS with master...
20:01:50.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:50.605345 12087 heartbeater.cc:511] Master 127.10.246.252:36187 requested a full tablet report, sending...
20:01:50.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:50.605778 12086 heartbeater.cc:511] Master 127.10.246.253:45131 requested a full tablet report, sending...
20:01:50.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:50.608060 11469 ts_manager.cc:194] Registered new tserver with Master: 762edfeaedf7488b81ac50d1e39edaa1 (127.10.246.195:45981)
20:01:50.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:50.608806 12089 heartbeater.cc:347] Connected to a master server at 127.10.246.254:34977
20:01:50.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:50.609129 12089 heartbeater.cc:464] Registering TS with master...
20:01:50.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:50.609733 11469 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.246.195:34443
20:01:50.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:50.610036 12089 heartbeater.cc:511] Master 127.10.246.254:34977 requested a full tablet report, sending...
20:01:50.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:50.611783 11378 ts_manager.cc:194] Registered new tserver with Master: 762edfeaedf7488b81ac50d1e39edaa1 (127.10.246.195:45981)
20:01:50.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:50.613052 11284 ts_manager.cc:194] Registered new tserver with Master: 762edfeaedf7488b81ac50d1e39edaa1 (127.10.246.195:45981)
20:01:50.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:50.624270 11227 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
20:01:50.637 [DEBUG - main] (MiniKuduCluster.java:195) Response:
20:01:50.638 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
20:01:50.650 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.10.246.254"
port: 34977
}
bound_http_address {
host: "127.10.246.254"
port: 36839
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.10.246.253"
port: 45131
}
bound_http_address {
host: "127.10.246.253"
port: 45329
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.10.246.252"
port: 36187
}
bound_http_address {
host: "127.10.246.252"
port: 40689
}
}
}
20:01:50.656 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
20:01:50.660 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.10.246.193"
port: 40381
}
bound_http_address {
host: "127.10.246.193"
port: 41887
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.10.246.194"
port: 36153
}
bound_http_address {
host: "127.10.246.194"
port: 35437
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.10.246.195"
port: 45981
}
bound_http_address {
host: "127.10.246.195"
port: 37587
}
}
}
20:01:50.660 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
20:01:50.887 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
20:01:51.035 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskBduYyb/test-tmp
20:01:51.036 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
20:01:51.036 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskBduYyb/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
20:01:51.040 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
20:01:51.412 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:01:51.412 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
20:01:51.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:51.415757 12100 mini_oidc.cc:152] Starting JWKS server
20:01:51.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:51.419467 12100 webserver.cc:460] Webserver started at http://127.0.0.1:36235/ using document root <none> and password file <none>
20:01:51.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:51.419987 12100 mini_oidc.cc:189] Starting OIDC Discovery server
20:01:51.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:51.421824 12100 webserver.cc:460] Webserver started at http://127.0.0.1:33809/ using document root <none> and password file <none>
20:01:51.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
20:01:51.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/krb5kdc/principal' for realm 'KRBTEST.COM',
20:01:51.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
20:01:51.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 20:01:51 dist-test-slave-9gcw krb5kdc[12116](info): setting up network...
20:01:51.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
20:01:51.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 20:01:51 dist-test-slave-9gcw krb5kdc[12116](info): set up 2 sockets
20:01:51.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 20:01:51 dist-test-slave-9gcw krb5kdc[12116](info): commencing operation
20:01:51.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
20:01:51.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:51.614356 12087 heartbeater.cc:503] Master 127.10.246.252:36187 was elected leader, sending a full tablet report...
20:01:52.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:52.752275 12100 mini_kdc.cc:121] Time spent starting KDC: real 1.304s user 0.003s sys 0.025s
20:01:52.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
20:01:52.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:01:52.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
20:01:52.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
20:01:52.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:01:52.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
20:01:52.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
20:01:52.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:01:52.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
20:01:52.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:01:52.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/krb5kdc/test-user.keytab.
20:01:52.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/krb5kdc/test-user.keytab.
20:01:52.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 20:01:52 dist-test-slave-9gcw krb5kdc[12116](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786478512, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:01:52.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
20:01:52.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-11T20:01:52Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
20:01:52.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-11T20:01:52Z Disabled control of system clock
20:01:52.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.11.209.62@KRBTEST.COM; defaulting to no policy
20:01:52.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:01:52.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.11.209.62@KRBTEST.COM" created.
20:01:53.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:01:53.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.11.209.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/krb5kdc/kudu_127.11.209.62.keytab.
20:01:53.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.11.209.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/krb5kdc/kudu_127.11.209.62.keytab.
20:01:53.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:53.007395 12100 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskBduYyb/build/tsan/bin/kudu
20:01:53.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskBduYyb/build/tsan/bin/kudu
20:01:53.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-0/wal
20:01:53.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-0/data
20:01:53.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:01:53.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:01:53.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:01:53.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:53.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:53.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:53.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-0/logs
20:01:53.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-0/data/info.pb
20:01:53.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:53.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:01:53.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:01:53.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:01:53.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:01:53.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:53.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:53.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:53.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:01:53.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:01:53.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:01:53.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:01:53.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.209.62:35309
20:01:53.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.209.62
20:01:53.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:01:53.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/krb5kdc/kudu_127.11.209.62.keytab
20:01:53.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.209.62
20:01:53.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:01:53.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:01:53.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:01:53.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.209.20:34897
20:01:53.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:53.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:53.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:53.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:01:53.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.209.62:35309,127.11.209.61:41765,127.11.209.60:33937
20:01:53.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:01:53.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36235/jwks.json
20:01:53.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:01:53.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:53.355886 12132 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:01:53.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:53.356732 12132 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:01:53.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:53.357620 12132 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:01:53.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:53.393348 12132 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
20:01:53.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:53.393641 12132 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:01:53.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:53.394088 12132 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:01:53.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:53.394434 12132 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:01:53.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:53.394747 12132 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:01:53.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:53.395038 12132 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:01:53.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:53.395260 12132 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36235/jwks.json
20:01:53.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:53.432473 12132 master_runner.cc:386] Master server non-default flags:
20:01:53.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:53.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.209.20:34897
20:01:53.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:53.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:53.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-0/data
20:01:53.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-0/wal
20:01:53.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:01:53.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:01:53.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.209.62:35309,127.11.209.61:41765,127.11.209.60:33937
20:01:53.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/krb5kdc/kudu_127.11.209.62.keytab
20:01:53.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.209.62
20:01:53.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:53.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:53.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:01:53.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.209.62:35309
20:01:53.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:01:53.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:01:53.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:01:53.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36235/jwks.json
20:01:53.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:01:53.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:01:53.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:01:53.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:53.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:53.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-0/data/info.pb
20:01:53.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.209.62
20:01:53.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:01:53.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:01:53.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:53.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:01:53.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:01:53.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:53.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-0/logs
20:01:53.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:53.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:01:53.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:01:53.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:01:53.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:01:53.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:01:53.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:01:53.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 18:54:03 UTC on ccycloud.quasar-qqxtnc.root.comops.site
20:01:53.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4581135
20:01:53.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:01:53.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:53.434142 12132 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:01:53.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:53.435421 12132 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:01:53.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:53.455247 12140 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:54.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:53.455454 12139 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:54.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:53.455268 12142 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:54.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:54.859241 12141 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1401 milliseconds
20:01:54.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:54.859367 12132 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:01:54.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:54.860594 12132 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:01:54.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:54.863857 12132 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:01:54.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:54.865260 12132 hybrid_clock.cc:648] HybridClock initialized: now 1786478514865204 us; error 103 us; skew 500 ppm
20:01:54.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:54.866062 12132 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:01:54.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 20:01:54 dist-test-slave-9gcw krb5kdc[12116](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786478514, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.209.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:01:54.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:54.873314 12132 init.cc:377] Logged in from keytab as kudu/127.11.209.62@KRBTEST.COM (short username kudu)
20:01:54.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:54.879413 12132 webserver.cc:460] Webserver started at http://127.11.209.62:38183/ using document root <none> and password file <none>
20:01:54.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:54.880519 12132 fs_manager.cc:359] Metadata directory not provided
20:01:54.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:54.880766 12132 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:01:54.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:54.881376 12132 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:01:54.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:54.886433 12132 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-0/data/instance:
20:01:54.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ec327abf8690480e82493bad20d856ad"
20:01:54.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 20:01:54 on dist-test-slave-9gcw"
20:01:54.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:54.887517 12132 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-0/wal/instance:
20:01:54.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ec327abf8690480e82493bad20d856ad"
20:01:54.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 20:01:54 on dist-test-slave-9gcw"
20:01:54.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:54.895428 12132 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
20:01:54.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:54.901760 12155 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:01:54.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:54.902985 12132 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
20:01:54.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:54.903406 12132 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-0/data,/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-0/wal
20:01:54.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ec327abf8690480e82493bad20d856ad"
20:01:54.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 20:01:54 on dist-test-slave-9gcw"
20:01:54.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:54.903782 12132 fs_report.cc:383] FS layout report
20:01:54.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:01:54.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-0/wal
20:01:54.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-0/wal
20:01:54.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-0/data/data
20:01:54.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:01:54.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:01:54.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:01:54.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:01:54.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:01:54.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:01:54.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:01:54.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:01:54.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:01:54.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:01:54.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:01:54.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:54.948473 12132 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:01:54.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:54.954833 12132 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:01:54.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:54.955448 12132 kserver.cc:163] Server-wide thread pool size limit: 3276
20:01:55.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:55.034153 12132 rpc_server.cc:307] RPC server started. Bound to: 127.11.209.62:35309
20:01:55.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:55.034240 12217 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.11.209.62:35309 every 8 connection(s)
20:01:55.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:55.037055 12132 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-0/data/info.pb
20:01:55.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:55.039849 12100 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskBduYyb/build/tsan/bin/kudu as pid 12132
20:01:55.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:55.040534 12100 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-0/wal/instance
20:01:55.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:55.044845 12218 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:01:55.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:55.063759 12218 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.209.62" port: 35309 } has no permanent_uuid. Determining permanent_uuid...
20:01:55.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.11.209.61@KRBTEST.COM; defaulting to no policy
20:01:55.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:01:55.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.11.209.61@KRBTEST.COM" created.
20:01:55.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 20:01:55 dist-test-slave-9gcw krb5kdc[12116](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786478514, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.209.62@KRBTEST.COM for kudu/127.11.209.62@KRBTEST.COM
20:01:55.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:01:55.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.11.209.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/krb5kdc/kudu_127.11.209.61.keytab.
20:01:55.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.11.209.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/krb5kdc/kudu_127.11.209.61.keytab.
20:01:55.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:55.102408 12100 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskBduYyb/build/tsan/bin/kudu
20:01:55.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskBduYyb/build/tsan/bin/kudu
20:01:55.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-1/wal
20:01:55.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-1/data
20:01:55.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:01:55.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:01:55.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:01:55.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:55.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:55.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:55.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-1/logs
20:01:55.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-1/data/info.pb
20:01:55.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:55.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:01:55.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:01:55.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:01:55.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:01:55.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:55.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:55.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:55.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:01:55.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:01:55.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:01:55.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:01:55.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.209.61:41765
20:01:55.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.209.61
20:01:55.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:01:55.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/krb5kdc/kudu_127.11.209.61.keytab
20:01:55.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.209.61
20:01:55.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:01:55.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:01:55.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:01:55.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.209.20:34897
20:01:55.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:55.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:55.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:55.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:01:55.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.209.62:35309,127.11.209.61:41765,127.11.209.60:33937
20:01:55.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:01:55.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36235/jwks.json
20:01:55.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:01:55.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:55.108729 12218 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.209.61" port: 41765 } has no permanent_uuid. Determining permanent_uuid...
20:01:55.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:55.111761 12156 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.11.209.61:41765: connect: Connection refused (error 111)
20:01:55.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:55.116020 12218 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.209.61:41765: Network error: Client connection negotiation failed: client connection to 127.11.209.61:41765: connect: Connection refused (error 111)
20:01:55.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:55.165546 12218 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.209.61" port: 41765 } attempt: 1
20:01:55.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:55.169728 12218 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.209.61:41765: Network error: Client connection negotiation failed: client connection to 127.11.209.61:41765: connect: Connection refused (error 111)
20:01:55.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:55.238168 12218 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.209.61" port: 41765 } attempt: 2
20:01:55.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:55.242501 12218 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.209.61:41765: Network error: Client connection negotiation failed: client connection to 127.11.209.61:41765: connect: Connection refused (error 111)
20:01:55.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:55.333932 12218 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.209.61" port: 41765 } attempt: 3
20:01:55.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:55.338086 12218 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.209.61:41765: Network error: Client connection negotiation failed: client connection to 127.11.209.61:41765: connect: Connection refused (error 111)
20:01:55.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:55.464646 12227 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:01:55.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:55.465384 12227 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:01:55.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:55.466109 12227 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:01:55.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:55.481510 12218 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.209.61" port: 41765 } attempt: 4
20:01:55.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:55.486094 12218 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.209.61:41765: Network error: Client connection negotiation failed: client connection to 127.11.209.61:41765: connect: Connection refused (error 111)
20:01:55.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:55.501197 12227 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
20:01:55.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:55.501509 12227 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:01:55.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:55.501915 12227 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:01:55.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:55.502252 12227 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:01:55.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:55.502700 12227 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:01:55.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:55.503151 12227 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:01:55.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:55.503504 12227 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36235/jwks.json
20:01:55.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:55.547457 12227 master_runner.cc:386] Master server non-default flags:
20:01:55.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:55.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.209.20:34897
20:01:55.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:55.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:55.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-1/data
20:01:55.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-1/wal
20:01:55.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:01:55.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:01:55.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.209.62:35309,127.11.209.61:41765,127.11.209.60:33937
20:01:55.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/krb5kdc/kudu_127.11.209.61.keytab
20:01:55.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.209.61
20:01:55.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:55.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:55.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:01:55.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.209.61:41765
20:01:55.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:01:55.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:01:55.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:01:55.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36235/jwks.json
20:01:55.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:01:55.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:01:55.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:01:55.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:55.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:55.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-1/data/info.pb
20:01:55.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.209.61
20:01:55.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:01:55.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:01:55.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:55.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:01:55.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:01:55.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:55.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-1/logs
20:01:55.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:55.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:01:55.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:01:55.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:01:55.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:01:55.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:01:55.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:01:55.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 18:54:03 UTC on ccycloud.quasar-qqxtnc.root.comops.site
20:01:55.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4581135
20:01:55.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:01:55.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:55.549273 12227 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:01:55.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:55.550343 12227 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:01:55.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:55.570871 12237 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:55.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:55.785579 12218 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.209.61" port: 41765 } attempt: 5
20:01:55.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:55.793870 12218 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.209.61:41765: Network error: Client connection negotiation failed: client connection to 127.11.209.61:41765: connect: Connection refused (error 111)
20:01:56.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:56.341483 12218 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.209.61" port: 41765 } attempt: 6
20:01:56.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:56.356714 12218 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.209.61:41765: Network error: Client connection negotiation failed: client connection to 127.11.209.61:41765: connect: Connection refused (error 111)
20:01:56.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:55.570866 12236 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:56.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:55.571120 12239 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:56.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:56.925935 12238 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1351 milliseconds
20:01:56.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:56.926136 12227 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:01:56.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:56.927585 12227 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:01:56.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:56.930986 12227 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:01:56.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:56.932461 12227 hybrid_clock.cc:648] HybridClock initialized: now 1786478516932414 us; error 66 us; skew 500 ppm
20:01:56.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:56.933512 12227 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:01:56.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 20:01:56 dist-test-slave-9gcw krb5kdc[12116](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786478516, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.209.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:01:56.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:56.946993 12227 init.cc:377] Logged in from keytab as kudu/127.11.209.61@KRBTEST.COM (short username kudu)
20:01:56.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:56.952702 12227 webserver.cc:460] Webserver started at http://127.11.209.61:38865/ using document root <none> and password file <none>
20:01:56.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:56.953769 12227 fs_manager.cc:359] Metadata directory not provided
20:01:56.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:56.954015 12227 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:01:56.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:56.954553 12227 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:01:56.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:56.959803 12227 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-1/data/instance:
20:01:56.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e042378830b94a6e8d9826c011220f98"
20:01:56.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 20:01:56 on dist-test-slave-9gcw"
20:01:56.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:56.961042 12227 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-1/wal/instance:
20:01:56.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e042378830b94a6e8d9826c011220f98"
20:01:56.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 20:01:56 on dist-test-slave-9gcw"
20:01:56.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:56.970470 12227 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.007s sys 0.001s
20:01:56.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:56.976663 12252 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:01:56.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:56.977967 12227 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
20:01:56.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:56.978262 12227 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-1/data,/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-1/wal
20:01:56.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e042378830b94a6e8d9826c011220f98"
20:01:56.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 20:01:56 on dist-test-slave-9gcw"
20:01:56.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:56.978557 12227 fs_report.cc:383] FS layout report
20:01:56.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:01:56.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-1/wal
20:01:56.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-1/wal
20:01:56.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-1/data/data
20:01:56.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:01:56.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:01:56.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:01:56.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:01:56.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:01:56.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:01:56.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:01:56.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:01:56.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:01:56.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:01:56.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:01:57.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:57.066408 12227 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:01:57.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:57.072695 12227 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:01:57.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:57.073127 12227 kserver.cc:163] Server-wide thread pool size limit: 3276
20:01:57.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:57.152724 12227 rpc_server.cc:307] RPC server started. Bound to: 127.11.209.61:41765
20:01:57.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:57.152925 12314 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.11.209.61:41765 every 8 connection(s)
20:01:57.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:57.157670 12227 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-1/data/info.pb
20:01:57.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:57.161608 12100 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskBduYyb/build/tsan/bin/kudu as pid 12227
20:01:57.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:57.162186 12100 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-1/wal/instance
20:01:57.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:57.164350 12316 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:01:57.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:57.181370 12316 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.209.62" port: 35309 } has no permanent_uuid. Determining permanent_uuid...
20:01:57.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.11.209.60@KRBTEST.COM; defaulting to no policy
20:01:57.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:01:57.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.11.209.60@KRBTEST.COM" created.
20:01:57.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 20:01:57 dist-test-slave-9gcw krb5kdc[12116](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786478516, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.209.61@KRBTEST.COM for kudu/127.11.209.62@KRBTEST.COM
20:01:57.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:01:57.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.11.209.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/krb5kdc/kudu_127.11.209.60.keytab.
20:01:57.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.11.209.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/krb5kdc/kudu_127.11.209.60.keytab.
20:01:57.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:57.220118 12316 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.209.61" port: 41765 } has no permanent_uuid. Determining permanent_uuid...
20:01:57.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:57.220338 12100 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskBduYyb/build/tsan/bin/kudu
20:01:57.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskBduYyb/build/tsan/bin/kudu
20:01:57.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-2/wal
20:01:57.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-2/data
20:01:57.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:01:57.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:01:57.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:01:57.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:57.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:57.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:57.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-2/logs
20:01:57.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-2/data/info.pb
20:01:57.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:57.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:01:57.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:01:57.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:01:57.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:01:57.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:57.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:57.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:57.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:01:57.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:01:57.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:01:57.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:01:57.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.209.60:33937
20:01:57.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.209.60
20:01:57.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:01:57.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/krb5kdc/kudu_127.11.209.60.keytab
20:01:57.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.209.60
20:01:57.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:01:57.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:01:57.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:01:57.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.209.20:34897
20:01:57.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:57.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:57.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:57.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:01:57.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.209.62:35309,127.11.209.61:41765,127.11.209.60:33937
20:01:57.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:01:57.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36235/jwks.json
20:01:57.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:01:57.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 20:01:57 dist-test-slave-9gcw krb5kdc[12116](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786478516, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.209.61@KRBTEST.COM for kudu/127.11.209.61@KRBTEST.COM
20:01:57.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:57.256711 12316 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.209.60" port: 33937 } has no permanent_uuid. Determining permanent_uuid...
20:01:57.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:57.259670 12254 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.11.209.60:33937: connect: Connection refused (error 111)
20:01:57.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:57.264727 12316 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.209.60:33937: Network error: Client connection negotiation failed: client connection to 127.11.209.60:33937: connect: Connection refused (error 111)
20:01:57.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:57.314443 12316 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.209.60" port: 33937 } attempt: 1
20:01:57.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:57.318823 12316 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.209.60:33937: Network error: Client connection negotiation failed: client connection to 127.11.209.60:33937: connect: Connection refused (error 111)
20:01:57.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:57.387334 12316 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.209.60" port: 33937 } attempt: 2
20:01:57.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:57.391388 12316 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.209.60:33937: Network error: Client connection negotiation failed: client connection to 127.11.209.60:33937: connect: Connection refused (error 111)
20:01:57.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:57.417266 12218 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.209.61" port: 41765 } attempt: 7
20:01:57.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 20:01:57 dist-test-slave-9gcw krb5kdc[12116](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786478514, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.209.62@KRBTEST.COM for kudu/127.11.209.61@KRBTEST.COM
20:01:57.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:57.445385 12218 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.209.60" port: 33937 } has no permanent_uuid. Determining permanent_uuid...
20:01:57.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:57.451179 12218 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.209.60:33937: Network error: Client connection negotiation failed: client connection to 127.11.209.60:33937: connect: Connection refused (error 111)
20:01:57.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:57.482913 12316 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.209.60" port: 33937 } attempt: 3
20:01:57.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:57.486887 12316 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.209.60:33937: Network error: Client connection negotiation failed: client connection to 127.11.209.60:33937: connect: Connection refused (error 111)
20:01:57.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:57.509719 12218 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.209.60" port: 33937 } attempt: 1
20:01:57.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:57.513880 12218 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.209.60:33937: Network error: Client connection negotiation failed: client connection to 127.11.209.60:33937: connect: Connection refused (error 111)
20:01:57.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:57.595415 12218 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.209.60" port: 33937 } attempt: 2
20:01:57.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:57.599529 12218 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.209.60:33937: Network error: Client connection negotiation failed: client connection to 127.11.209.60:33937: connect: Connection refused (error 111)
20:01:57.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:57.613289 12324 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:01:57.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:57.613869 12324 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:01:57.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:57.614461 12324 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:01:57.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:57.630255 12316 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.209.60" port: 33937 } attempt: 4
20:01:57.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:57.637782 12316 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.209.60:33937: Network error: Client connection negotiation failed: client connection to 127.11.209.60:33937: connect: Connection refused (error 111)
20:01:57.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:57.653436 12324 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
20:01:57.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:57.653780 12324 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:01:57.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:57.654232 12324 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:01:57.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:57.654587 12324 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:01:57.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:57.654919 12324 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:01:57.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:57.655131 12324 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:01:57.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:57.655274 12324 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36235/jwks.json
20:01:57.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:57.685070 12218 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.209.60" port: 33937 } attempt: 3
20:01:57.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:57.689651 12218 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.209.60:33937: Network error: Client connection negotiation failed: client connection to 127.11.209.60:33937: connect: Connection refused (error 111)
20:01:57.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:57.696167 12324 master_runner.cc:386] Master server non-default flags:
20:01:57.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:57.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.209.20:34897
20:01:57.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:57.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:57.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-2/data
20:01:57.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-2/wal
20:01:57.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:01:57.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:01:57.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.209.62:35309,127.11.209.61:41765,127.11.209.60:33937
20:01:57.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/krb5kdc/kudu_127.11.209.60.keytab
20:01:57.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.209.60
20:01:57.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:57.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:57.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:01:57.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.209.60:33937
20:01:57.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:01:57.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:01:57.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:01:57.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36235/jwks.json
20:01:57.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:01:57.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:01:57.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:01:57.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:57.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:57.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-2/data/info.pb
20:01:57.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.209.60
20:01:57.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:01:57.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:01:57.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:57.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:01:57.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:01:57.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:57.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-2/logs
20:01:57.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:57.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:01:57.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:01:57.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:01:57.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:01:57.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:01:57.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:01:57.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 18:54:03 UTC on ccycloud.quasar-qqxtnc.root.comops.site
20:01:57.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4581135
20:01:57.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:01:57.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:57.697808 12324 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:01:57.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:57.699067 12324 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:01:57.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:57.718922 12337 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:57.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:57.830081 12218 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.209.60" port: 33937 } attempt: 4
20:01:57.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:57.835454 12218 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.209.60:33937: Network error: Client connection negotiation failed: client connection to 127.11.209.60:33937: connect: Connection refused (error 111)
20:01:57.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:57.937258 12316 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.209.60" port: 33937 } attempt: 5
20:01:57.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:57.942349 12316 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.209.60:33937: Network error: Client connection negotiation failed: client connection to 127.11.209.60:33937: connect: Connection refused (error 111)
20:01:58.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:58.118920 12218 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.209.60" port: 33937 } attempt: 5
20:01:58.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:58.124279 12218 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.209.60:33937: Network error: Client connection negotiation failed: client connection to 127.11.209.60:33937: connect: Connection refused (error 111)
20:01:58.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:58.489866 12316 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.209.60" port: 33937 } attempt: 6
20:01:58.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:58.498090 12316 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.209.60:33937: Network error: Client connection negotiation failed: client connection to 127.11.209.60:33937: connect: Connection refused (error 111)
20:01:58.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:58.676788 12218 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.209.60" port: 33937 } attempt: 6
20:01:58.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:58.685391 12218 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.209.60:33937: Network error: Client connection negotiation failed: client connection to 127.11.209.60:33937: connect: Connection refused (error 111)
20:01:59.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:57.721787 12334 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:59.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:57.718945 12335 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:59.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:59.036741 12324 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:01:59.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:59.036279 12336 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
20:01:59.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:59.041026 12324 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:01:59.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:59.044361 12324 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:01:59.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:59.045919 12324 hybrid_clock.cc:648] HybridClock initialized: now 1786478519045852 us; error 79 us; skew 500 ppm
20:01:59.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:59.047237 12324 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:01:59.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 20:01:59 dist-test-slave-9gcw krb5kdc[12116](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786478519, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.209.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:01:59.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:59.053942 12324 init.cc:377] Logged in from keytab as kudu/127.11.209.60@KRBTEST.COM (short username kudu)
20:01:59.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:59.060161 12324 webserver.cc:460] Webserver started at http://127.11.209.60:37663/ using document root <none> and password file <none>
20:01:59.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:59.061328 12324 fs_manager.cc:359] Metadata directory not provided
20:01:59.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:59.061640 12324 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:01:59.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:59.062282 12324 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:01:59.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:59.067759 12324 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-2/data/instance:
20:01:59.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "48846ff3bd7e4bd999c2d47fcfe534a9"
20:01:59.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 20:01:59 on dist-test-slave-9gcw"
20:01:59.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:59.068889 12324 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-2/wal/instance:
20:01:59.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "48846ff3bd7e4bd999c2d47fcfe534a9"
20:01:59.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 20:01:59 on dist-test-slave-9gcw"
20:01:59.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:59.077702 12324 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.006s sys 0.001s
20:01:59.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:59.084519 12353 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:01:59.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:59.085778 12324 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
20:01:59.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:59.086220 12324 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-2/data,/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-2/wal
20:01:59.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "48846ff3bd7e4bd999c2d47fcfe534a9"
20:01:59.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 20:01:59 on dist-test-slave-9gcw"
20:01:59.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:59.086614 12324 fs_report.cc:383] FS layout report
20:01:59.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:01:59.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-2/wal
20:01:59.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-2/wal
20:01:59.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-2/data/data
20:01:59.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:01:59.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:01:59.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:01:59.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:01:59.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:01:59.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:01:59.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:01:59.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:01:59.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:01:59.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:01:59.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:01:59.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:59.160192 12324 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:01:59.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:59.167366 12324 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:01:59.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:59.167966 12324 kserver.cc:163] Server-wide thread pool size limit: 3276
20:01:59.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:59.257676 12324 rpc_server.cc:307] RPC server started. Bound to: 127.11.209.60:33937
20:01:59.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:59.257756 12418 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.11.209.60:33937 every 8 connection(s)
20:01:59.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:59.261816 12324 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-2/data/info.pb
20:01:59.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:59.269426 12420 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:01:59.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:59.269865 12100 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskBduYyb/build/tsan/bin/kudu as pid 12324
20:01:59.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:59.270392 12100 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-2/wal/instance
20:01:59.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:59.285362 12420 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.209.62" port: 35309 } has no permanent_uuid. Determining permanent_uuid...
20:01:59.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 20:01:59 dist-test-slave-9gcw krb5kdc[12116](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786478519, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.209.60@KRBTEST.COM for kudu/127.11.209.62@KRBTEST.COM
20:01:59.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 20:01:59 dist-test-slave-9gcw krb5kdc[12116](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786478512, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.11.209.60@KRBTEST.COM
20:01:59.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:59.327347 12420 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.209.61" port: 41765 } has no permanent_uuid. Determining permanent_uuid...
20:01:59.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 20:01:59 dist-test-slave-9gcw krb5kdc[12116](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786478519, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.209.60@KRBTEST.COM for kudu/127.11.209.61@KRBTEST.COM
20:01:59.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 20:01:59 dist-test-slave-9gcw krb5kdc[12116](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786478512, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.11.209.61@KRBTEST.COM
20:01:59.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:59.357726 12420 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.209.60" port: 33937 } has no permanent_uuid. Determining permanent_uuid...
20:01:59.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 20:01:59 dist-test-slave-9gcw krb5kdc[12116](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786478519, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.209.60@KRBTEST.COM for kudu/127.11.209.60@KRBTEST.COM
20:01:59.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 20:01:59 dist-test-slave-9gcw krb5kdc[12116](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786478512, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.11.209.62@KRBTEST.COM
20:01:59.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:59.392040 12420 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9: Bootstrap starting.
20:01:59.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:59.398480 12420 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9: Neither blocks nor log segments found. Creating new log.
20:01:59.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:59.400122 12100 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
20:01:59.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:59.400498 12420 log.cc:824] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9: Log is configured to *not* fsync() on all Append() calls
20:01:59.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:59.405632 12420 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9: No bootstrap required, opened a new log
20:01:59.409 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:01:59.411 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
20:01:59.425 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
20:01:59.426 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
20:01:59.429 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.11.209.62"
port: 35309
}
bound_http_address {
host: "127.11.209.62"
port: 38183
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.11.209.61"
port: 41765
}
bound_http_address {
host: "127.11.209.61"
port: 38865
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.11.209.60"
port: 33937
}
bound_http_address {
host: "127.11.209.60"
port: 37663
}
}
}
20:01:59.430 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
20:01:59.430 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:01:59.430 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
20:01:59.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:59.429181 12420 raft_consensus.cc:348] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ec327abf8690480e82493bad20d856ad" member_type: VOTER last_known_addr { host: "127.11.209.62" port: 35309 } } peers { permanent_uuid: "e042378830b94a6e8d9826c011220f98" member_type: VOTER last_known_addr { host: "127.11.209.61" port: 41765 } } peers { permanent_uuid: "48846ff3bd7e4bd999c2d47fcfe534a9" member_type: VOTER last_known_addr { host: "127.11.209.60" port: 33937 } }
20:01:59.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:59.432281 12420 raft_consensus.cc:374] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:01:59.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:59.432720 12420 raft_consensus.cc:729] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 48846ff3bd7e4bd999c2d47fcfe534a9, State: Initialized, Role: FOLLOWER
20:01:59.433 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
20:01:59.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:59.433845 12420 consensus_queue.cc:260] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ec327abf8690480e82493bad20d856ad" member_type: VOTER last_known_addr { host: "127.11.209.62" port: 35309 } } peers { permanent_uuid: "e042378830b94a6e8d9826c011220f98" member_type: VOTER last_known_addr { host: "127.11.209.61" port: 41765 } } peers { permanent_uuid: "48846ff3bd7e4bd999c2d47fcfe534a9" member_type: VOTER last_known_addr { host: "127.11.209.60" port: 33937 } }
20:01:59.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:59.437470 12429 sys_catalog.cc:455] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ec327abf8690480e82493bad20d856ad" member_type: VOTER last_known_addr { host: "127.11.209.62" port: 35309 } } peers { permanent_uuid: "e042378830b94a6e8d9826c011220f98" member_type: VOTER last_known_addr { host: "127.11.209.61" port: 41765 } } peers { permanent_uuid: "48846ff3bd7e4bd999c2d47fcfe534a9" member_type: VOTER last_known_addr { host: "127.11.209.60" port: 33937 } } }
20:01:59.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:59.438553 12429 sys_catalog.cc:458] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9 [sys.catalog]: This master's current role is: FOLLOWER
20:01:59.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:59.439545 12420 sys_catalog.cc:564] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9 [sys.catalog]: configured and running, proceeding with master startup.
20:01:59.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:59.475489 12443 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:01:59.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:59.475907 12443 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:01:59.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 20:01:59 dist-test-slave-9gcw krb5kdc[12116](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786478519, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:01:59.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
20:01:59.490 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:01:59.493 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/krb5kdc/krb5cc
20:01:59.502 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
20:01:59.503 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/krb5kdc/krb5cc
20:01:59.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:59.465742 12324 master_runner.cc:186] Error getting master registration for 127.11.209.62:35309: OK, instance_id { permanent_uuid: "ec327abf8690480e82493bad20d856ad" instance_seqno: 1786478515013138 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:01:59.504 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
20:01:59.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:59.504061 12324 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:01:59.511 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
20:01:59.513 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
20:01:59.514 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
20:01:59.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 20:01:59 dist-test-slave-9gcw krb5kdc[12116](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786478519, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:01:59.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
20:01:59.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:59.558704 12316 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.209.60" port: 33937 } attempt: 7
20:01:59.559 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:01:59.560 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/krb5kdc/krb5cc
20:01:59.562 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
20:01:59.565 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.11.209.61:41765
20:01:59.567 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
20:01:59.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:59.568578 12100 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskBduYyb/build/tsan/bin/kudu with pid 12227
20:01:59.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:01:59.598815 12422 negotiation.cc:337] Failed RPC negotiation. Trace:
20:01:59.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0811 20:01:59.560958 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:56070 (local address 127.11.209.60:33937)
20:01:59.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0811 20:01:59.561652 (+ 694us) server_negotiation.cc:206] Beginning negotiation
20:01:59.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0811 20:01:59.561685 (+ 33us) server_negotiation.cc:399] Waiting for connection header
20:01:59.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0811 20:01:59.563075 (+ 1390us) server_negotiation.cc:407] Connection header received
20:01:59.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0811 20:01:59.563608 (+ 533us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
20:01:59.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0811 20:01:59.563636 (+ 28us) server_negotiation.cc:461] Received NEGOTIATE request from client
20:01:59.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0811 20:01:59.563880 (+ 244us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
20:01:59.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0811 20:01:59.564027 (+ 147us) server_negotiation.cc:226] Negotiated authn=SASL
20:01:59.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0811 20:01:59.566753 (+ 2726us) server_negotiation.cc:365] Received TLS_HANDSHAKE NegotiatePB request
20:01:59.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0811 20:01:59.568070 (+ 1317us) server_negotiation.cc:377] Sending TLS_HANDSHAKE NegotiatePB response
20:01:59.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0811 20:01:59.595904 (+ 27834us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:56070: BlockingRecv error: recv got EOF from 127.0.0.1:56070 (error 108)
20:01:59.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":141}
20:01:59.601 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:01:59.602 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.11.209.60:33937
20:01:59.602 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
20:01:59.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:59.602692 12100 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskBduYyb/build/tsan/bin/kudu with pid 12324
20:01:59.639 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:01:59.640 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.11.209.62:35309
20:01:59.640 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
20:01:59.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:59.640739 12100 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskBduYyb/build/tsan/bin/kudu with pid 12132
20:01:59.676 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:01:59.677 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.11.209.61:41765
20:01:59.679 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
20:01:59.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:01:59.681721 12100 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskBduYyb/build/tsan/bin/kudu
20:01:59.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskBduYyb/build/tsan/bin/kudu
20:01:59.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-1/wal
20:01:59.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-1/data
20:01:59.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:01:59.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:01:59.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:01:59.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:59.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:59.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:59.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-1/logs
20:01:59.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-1/data/info.pb
20:01:59.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:59.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:01:59.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:01:59.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:01:59.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:01:59.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:59.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:59.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:59.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:01:59.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:01:59.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:01:59.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:01:59.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.209.61:41765
20:01:59.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.209.61
20:01:59.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38865
20:01:59.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/krb5kdc/kudu_127.11.209.61.keytab
20:01:59.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.209.61
20:01:59.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:01:59.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:01:59.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:01:59.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.209.20:34897
20:01:59.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:59.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:59.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:59.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:01:59.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.209.62:35309,127.11.209.61:41765,127.11.209.60:33937
20:01:59.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:01:59.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36235/jwks.json
20:01:59.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:02:00.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:00.027330 12450 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:02:00.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:00.027915 12450 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:02:00.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:00.028508 12450 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:02:00.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:00.063167 12450 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
20:02:00.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:00.063421 12450 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:02:00.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:00.063691 12450 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:02:00.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:00.064034 12450 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:02:00.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:00.064244 12450 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:02:00.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:00.064424 12450 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:02:00.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:00.064556 12450 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36235/jwks.json
20:02:00.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:00.104146 12450 master_runner.cc:386] Master server non-default flags:
20:02:00.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:02:00.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.209.20:34897
20:02:00.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:02:00.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:02:00.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-1/data
20:02:00.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-1/wal
20:02:00.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:02:00.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:02:00.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.209.62:35309,127.11.209.61:41765,127.11.209.60:33937
20:02:00.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/krb5kdc/kudu_127.11.209.61.keytab
20:02:00.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.209.61
20:02:00.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:02:00.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:02:00.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:02:00.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.209.61:41765
20:02:00.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:02:00.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:02:00.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:02:00.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36235/jwks.json
20:02:00.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:02:00.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:02:00.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:02:00.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:02:00.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:02:00.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-1/data/info.pb
20:02:00.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.209.61
20:02:00.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38865
20:02:00.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:02:00.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:02:00.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:02:00.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:02:00.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:02:00.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-1/logs
20:02:00.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:02:00.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:02:00.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:02:00.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:02:00.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:02:00.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:02:00.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:02:00.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 18:54:03 UTC on ccycloud.quasar-qqxtnc.root.comops.site
20:02:00.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4581135
20:02:00.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:02:00.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:00.105427 12450 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:02:00.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:00.106518 12450 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:02:00.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:00.126405 12459 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:02:01.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:00.127192 12461 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:02:01.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:00.128198 12450 server_base.cc:1034] running on GCE node
20:02:01.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:00.127449 12458 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:02:01.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:01.414901 12450 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:02:01.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:01.418474 12450 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:02:01.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:01.420024 12450 hybrid_clock.cc:648] HybridClock initialized: now 1786478521419901 us; error 126 us; skew 500 ppm
20:02:01.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:01.421067 12450 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:02:01.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 20:02:01 dist-test-slave-9gcw krb5kdc[12116](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786478521, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.209.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:02:01.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:01.428009 12450 init.cc:377] Logged in from keytab as kudu/127.11.209.61@KRBTEST.COM (short username kudu)
20:02:01.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:01.434904 12450 webserver.cc:460] Webserver started at http://127.11.209.61:38865/ using document root <none> and password file <none>
20:02:01.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:01.436234 12450 fs_manager.cc:359] Metadata directory not provided
20:02:01.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:01.436470 12450 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:02:01.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:01.445663 12450 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.007s sys 0.001s
20:02:01.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:01.450485 12475 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:02:01.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:01.451617 12450 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.000s sys 0.005s
20:02:01.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:01.452013 12450 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-1/data,/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-1/wal
20:02:01.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e042378830b94a6e8d9826c011220f98"
20:02:01.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 20:01:56 on dist-test-slave-9gcw"
20:02:01.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:01.454675 12450 fs_report.cc:383] FS layout report
20:02:01.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:02:01.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-1/wal
20:02:01.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-1/wal
20:02:01.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-1/data/data
20:02:01.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:02:01.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:02:01.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:02:01.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:02:01.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:02:01.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:02:01.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:02:01.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:02:01.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:02:01.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:02:01.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:02:01.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:01.503130 12450 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:02:01.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:01.509274 12450 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:02:01.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:01.509843 12450 kserver.cc:163] Server-wide thread pool size limit: 3276
20:02:01.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:01.599403 12450 rpc_server.cc:307] RPC server started. Bound to: 127.11.209.61:41765
20:02:01.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:01.599583 12533 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.11.209.61:41765 every 8 connection(s)
20:02:01.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:01.602520 12450 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-1/data/info.pb
20:02:01.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:01.608049 12100 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskBduYyb/build/tsan/bin/kudu as pid 12450
20:02:01.609 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:02:01.609 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.11.209.60:33937
20:02:01.609 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
20:02:01.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:01.610994 12100 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskBduYyb/build/tsan/bin/kudu
20:02:01.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskBduYyb/build/tsan/bin/kudu
20:02:01.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-2/wal
20:02:01.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-2/data
20:02:01.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:02:01.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:02:01.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:02:01.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:02:01.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:02:01.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:02:01.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-2/logs
20:02:01.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-2/data/info.pb
20:02:01.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:02:01.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:02:01.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:02:01.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:02:01.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:02:01.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:02:01.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:02:01.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:02:01.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:02:01.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:02:01.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:02:01.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:02:01.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.209.60:33937
20:02:01.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.209.60
20:02:01.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37663
20:02:01.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/krb5kdc/kudu_127.11.209.60.keytab
20:02:01.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.209.60
20:02:01.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:02:01.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:02:01.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:02:01.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.209.20:34897
20:02:01.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:02:01.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:02:01.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:02:01.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:02:01.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.209.62:35309,127.11.209.61:41765,127.11.209.60:33937
20:02:01.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:02:01.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36235/jwks.json
20:02:01.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:02:01.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:01.616629 12534 sys_catalog.cc:263] Verifying existing consensus state
20:02:01.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260811 20:02:01.617599 12534 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
20:02:01.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:01.618093 12450 master.cc:537] Master@127.11.209.61:41765 shutting down...
20:02:01.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:01.728461 12450 master.cc:552] Master@127.11.209.61:41765 shutdown complete.
20:02:01.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
20:02:01.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:01.964447 12536 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:02:01.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:01.965380 12536 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:02:01.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:01.966107 12536 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:02:01.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:01.998561 12536 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
20:02:01.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:01.998808 12536 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:02:01.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:01.999068 12536 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:02:01.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:01.999255 12536 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:02:01.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:01.999461 12536 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:02:01.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:01.999637 12536 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:02:01.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:01.999789 12536 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36235/jwks.json
20:02:02.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:02.037986 12536 master_runner.cc:386] Master server non-default flags:
20:02:02.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:02:02.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.209.20:34897
20:02:02.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:02:02.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:02:02.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-2/data
20:02:02.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-2/wal
20:02:02.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:02:02.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:02:02.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.209.62:35309,127.11.209.61:41765,127.11.209.60:33937
20:02:02.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/krb5kdc/kudu_127.11.209.60.keytab
20:02:02.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.209.60
20:02:02.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:02:02.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:02:02.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:02:02.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.209.60:33937
20:02:02.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:02:02.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:02:02.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:02:02.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36235/jwks.json
20:02:02.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:02:02.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:02:02.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:02:02.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:02:02.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:02:02.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-2/data/info.pb
20:02:02.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.209.60
20:02:02.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37663
20:02:02.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:02:02.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:02:02.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:02:02.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:02:02.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:02:02.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-2/logs
20:02:02.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:02:02.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:02:02.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:02:02.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:02:02.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:02:02.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:02:02.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:02:02.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 18:54:03 UTC on ccycloud.quasar-qqxtnc.root.comops.site
20:02:02.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4581135
20:02:02.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:02:02.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:02.039196 12536 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:02:02.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:02.040423 12536 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:02:03.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:02.058979 12543 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:02:03.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:02.058986 12546 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:02:03.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:02.060833 12536 server_base.cc:1034] running on GCE node
20:02:03.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:02.058940 12549 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:02:03.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:03.331728 12536 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:02:03.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:03.335194 12536 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:02:03.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:03.336769 12536 hybrid_clock.cc:648] HybridClock initialized: now 1786478523336717 us; error 70 us; skew 500 ppm
20:02:03.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:03.337878 12536 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:02:03.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 20:02:03 dist-test-slave-9gcw krb5kdc[12116](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786478523, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.209.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:02:03.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:03.348682 12536 init.cc:377] Logged in from keytab as kudu/127.11.209.60@KRBTEST.COM (short username kudu)
20:02:03.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:03.354455 12536 webserver.cc:460] Webserver started at http://127.11.209.60:37663/ using document root <none> and password file <none>
20:02:03.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:03.355400 12536 fs_manager.cc:359] Metadata directory not provided
20:02:03.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:03.355707 12536 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:02:03.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:03.365028 12536 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.006s sys 0.001s
20:02:03.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:03.370304 12562 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:02:03.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:03.371666 12536 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
20:02:03.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:03.371997 12536 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-2/data,/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-2/wal
20:02:03.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "48846ff3bd7e4bd999c2d47fcfe534a9"
20:02:03.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 20:01:59 on dist-test-slave-9gcw"
20:02:03.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:03.374637 12536 fs_report.cc:383] FS layout report
20:02:03.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:02:03.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-2/wal
20:02:03.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-2/wal
20:02:03.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-2/data/data
20:02:03.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:02:03.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:02:03.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:02:03.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:02:03.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:02:03.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:02:03.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:02:03.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:02:03.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:02:03.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:02:03.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:02:03.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:03.413712 12536 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:02:03.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:03.420264 12536 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:02:03.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:03.420742 12536 kserver.cc:163] Server-wide thread pool size limit: 3276
20:02:03.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:03.508147 12536 rpc_server.cc:307] RPC server started. Bound to: 127.11.209.60:33937
20:02:03.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:03.508288 12618 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.11.209.60:33937 every 8 connection(s)
20:02:03.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:03.511355 12536 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-2/data/info.pb
20:02:03.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:03.522114 12100 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskBduYyb/build/tsan/bin/kudu as pid 12536
20:02:03.523 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:02:03.523 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.11.209.62:35309
20:02:03.523 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
20:02:03.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:03.523803 12620 sys_catalog.cc:263] Verifying existing consensus state
20:02:03.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:03.525254 12100 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskBduYyb/build/tsan/bin/kudu
20:02:03.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskBduYyb/build/tsan/bin/kudu
20:02:03.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-0/wal
20:02:03.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-0/data
20:02:03.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:02:03.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:02:03.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:02:03.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:02:03.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:02:03.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:02:03.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-0/logs
20:02:03.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-0/data/info.pb
20:02:03.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:02:03.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:02:03.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:02:03.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:02:03.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:02:03.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:02:03.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:02:03.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:02:03.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:02:03.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:02:03.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:02:03.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:02:03.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.209.62:35309
20:02:03.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.209.62
20:02:03.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38183
20:02:03.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/krb5kdc/kudu_127.11.209.62.keytab
20:02:03.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.209.62
20:02:03.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:02:03.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:02:03.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:02:03.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.209.20:34897
20:02:03.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:02:03.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:02:03.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:02:03.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:02:03.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.209.62:35309,127.11.209.61:41765,127.11.209.60:33937
20:02:03.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:02:03.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36235/jwks.json
20:02:03.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:02:03.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:03.530936 12620 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9: Bootstrap starting.
20:02:03.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:03.583808 12620 log.cc:824] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9: Log is configured to *not* fsync() on all Append() calls
20:02:03.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:03.588845 12620 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:02:03.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:03.589761 12620 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9: Bootstrap complete.
20:02:03.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:03.617623 12620 raft_consensus.cc:348] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ec327abf8690480e82493bad20d856ad" member_type: VOTER last_known_addr { host: "127.11.209.62" port: 35309 } } peers { permanent_uuid: "e042378830b94a6e8d9826c011220f98" member_type: VOTER last_known_addr { host: "127.11.209.61" port: 41765 } } peers { permanent_uuid: "48846ff3bd7e4bd999c2d47fcfe534a9" member_type: VOTER last_known_addr { host: "127.11.209.60" port: 33937 } }
20:02:03.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:03.620443 12620 raft_consensus.cc:374] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:02:03.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:03.620688 12620 raft_consensus.cc:729] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 48846ff3bd7e4bd999c2d47fcfe534a9, State: Initialized, Role: FOLLOWER
20:02:03.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:03.621526 12620 consensus_queue.cc:260] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ec327abf8690480e82493bad20d856ad" member_type: VOTER last_known_addr { host: "127.11.209.62" port: 35309 } } peers { permanent_uuid: "e042378830b94a6e8d9826c011220f98" member_type: VOTER last_known_addr { host: "127.11.209.61" port: 41765 } } peers { permanent_uuid: "48846ff3bd7e4bd999c2d47fcfe534a9" member_type: VOTER last_known_addr { host: "127.11.209.60" port: 33937 } }
20:02:03.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:03.624712 12625 sys_catalog.cc:455] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ec327abf8690480e82493bad20d856ad" member_type: VOTER last_known_addr { host: "127.11.209.62" port: 35309 } } peers { permanent_uuid: "e042378830b94a6e8d9826c011220f98" member_type: VOTER last_known_addr { host: "127.11.209.61" port: 41765 } } peers { permanent_uuid: "48846ff3bd7e4bd999c2d47fcfe534a9" member_type: VOTER last_known_addr { host: "127.11.209.60" port: 33937 } } }
20:02:03.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:03.625717 12625 sys_catalog.cc:458] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9 [sys.catalog]: This master's current role is: FOLLOWER
20:02:03.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:03.626634 12620 sys_catalog.cc:564] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9 [sys.catalog]: configured and running, proceeding with master startup.
20:02:03.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:03.669324 12638 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:02:03.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:03.669703 12638 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:02:03.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:03.676753 12570 tablet.cc:2369] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:02:03.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:03.670746 12536 master_runner.cc:186] Error getting master registration for 127.11.209.62:35309: Network error: Client connection negotiation failed: client connection to 127.11.209.62:35309: connect: Connection refused (error 111),
20:02:03.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:03.706866 12536 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:02:03.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:03.813710 12625 raft_consensus.cc:482] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:02:03.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:03.814389 12625 raft_consensus.cc:504] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ec327abf8690480e82493bad20d856ad" member_type: VOTER last_known_addr { host: "127.11.209.62" port: 35309 } } peers { permanent_uuid: "e042378830b94a6e8d9826c011220f98" member_type: VOTER last_known_addr { host: "127.11.209.61" port: 41765 } } peers { permanent_uuid: "48846ff3bd7e4bd999c2d47fcfe534a9" member_type: VOTER last_known_addr { host: "127.11.209.60" port: 33937 } }
20:02:03.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:03.819811 12566 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.11.209.62:35309: connect: Connection refused (error 111)
20:02:03.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:03.825246 12625 leader_election.cc:288] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ec327abf8690480e82493bad20d856ad (127.11.209.62:35309), e042378830b94a6e8d9826c011220f98 (127.11.209.61:41765)
20:02:03.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:03.828524 12566 leader_election.cc:334] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ec327abf8690480e82493bad20d856ad (127.11.209.62:35309): Network error: Client connection negotiation failed: client connection to 127.11.209.62:35309: connect: Connection refused (error 111)
20:02:03.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:03.830207 12563 leader_election.cc:334] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e042378830b94a6e8d9826c011220f98 (127.11.209.61:41765): Network error: Client connection negotiation failed: client connection to 127.11.209.61:41765: connect: Connection refused (error 111)
20:02:03.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:03.830649 12563 leader_election.cc:302] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 48846ff3bd7e4bd999c2d47fcfe534a9; no voters: e042378830b94a6e8d9826c011220f98, ec327abf8690480e82493bad20d856ad
20:02:03.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:03.831475 12625 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:02:03.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:03.905049 12622 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:02:03.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:03.905717 12622 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:02:03.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:03.906301 12622 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:02:03.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:03.940841 12622 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
20:02:03.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:03.941535 12622 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:02:03.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:03.941859 12622 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:02:03.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:03.942063 12622 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:02:03.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:03.942260 12622 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:02:03.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:03.942579 12622 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:02:03.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:03.942832 12622 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36235/jwks.json
20:02:03.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:03.983541 12622 master_runner.cc:386] Master server non-default flags:
20:02:03.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:02:03.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.209.20:34897
20:02:03.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:02:03.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:02:03.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-0/data
20:02:03.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-0/wal
20:02:03.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:02:03.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:02:03.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.209.62:35309,127.11.209.61:41765,127.11.209.60:33937
20:02:03.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/krb5kdc/kudu_127.11.209.62.keytab
20:02:03.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.209.62
20:02:03.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:02:03.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:02:03.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:02:03.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.209.62:35309
20:02:03.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:02:03.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:02:03.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:02:03.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36235/jwks.json
20:02:03.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:02:03.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:02:03.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:02:03.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:02:03.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:02:03.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-0/data/info.pb
20:02:03.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.209.62
20:02:03.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38183
20:02:03.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:02:03.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:02:03.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:02:03.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:02:03.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:02:03.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-0/logs
20:02:03.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:02:03.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:02:03.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:02:03.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:02:03.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:02:03.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:02:03.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:02:03.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 11 Aug 2026 18:54:03 UTC on ccycloud.quasar-qqxtnc.root.comops.site
20:02:03.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4581135
20:02:03.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:02:03.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:03.984746 12622 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:02:03.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:03.986122 12622 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:02:04.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:04.007531 12648 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:02:04.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:04.675861 12638 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:02:04.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:04.678030 12638 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:02:05.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:05.686458 12638 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:02:05.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:05.686825 12638 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:02:05.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:04.009075 12647 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:02:05.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:04.012385 12651 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:02:05.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:05.734781 12622 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:02:05.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:05.735520 12649 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1723 milliseconds
20:02:05.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:05.737450 12622 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:02:05.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:05.741591 12622 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:02:05.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:05.743440 12622 hybrid_clock.cc:648] HybridClock initialized: now 1786478525743370 us; error 80 us; skew 500 ppm
20:02:05.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:05.744829 12622 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:02:05.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 20:02:05 dist-test-slave-9gcw krb5kdc[12116](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786478525, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.209.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:02:05.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:05.754616 12622 init.cc:377] Logged in from keytab as kudu/127.11.209.62@KRBTEST.COM (short username kudu)
20:02:05.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:05.761132 12622 webserver.cc:460] Webserver started at http://127.11.209.62:38183/ using document root <none> and password file <none>
20:02:05.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:05.762197 12622 fs_manager.cc:359] Metadata directory not provided
20:02:05.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:05.762449 12622 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:02:05.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:05.771386 12622 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.004s sys 0.000s
20:02:05.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:05.776746 12661 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:02:05.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:05.778088 12622 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.001s sys 0.002s
20:02:05.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:05.778424 12622 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-0/data,/tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-0/wal
20:02:05.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ec327abf8690480e82493bad20d856ad"
20:02:05.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-11 20:01:54 on dist-test-slave-9gcw"
20:02:05.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:05.781121 12622 fs_report.cc:383] FS layout report
20:02:05.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:02:05.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-0/wal
20:02:05.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-0/wal
20:02:05.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-0/data/data
20:02:05.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:02:05.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:02:05.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:02:05.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:02:05.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:02:05.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:02:05.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:02:05.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:02:05.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:02:05.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:02:05.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:02:05.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:05.845525 12622 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:02:05.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:05.852985 12622 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:02:05.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:05.853686 12622 kserver.cc:163] Server-wide thread pool size limit: 3276
20:02:05.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:05.882020 12670 raft_consensus.cc:482] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:02:05.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:05.882944 12670 raft_consensus.cc:504] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ec327abf8690480e82493bad20d856ad" member_type: VOTER last_known_addr { host: "127.11.209.62" port: 35309 } } peers { permanent_uuid: "e042378830b94a6e8d9826c011220f98" member_type: VOTER last_known_addr { host: "127.11.209.61" port: 41765 } } peers { permanent_uuid: "48846ff3bd7e4bd999c2d47fcfe534a9" member_type: VOTER last_known_addr { host: "127.11.209.60" port: 33937 } }
20:02:05.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:05.894598 12670 leader_election.cc:288] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ec327abf8690480e82493bad20d856ad (127.11.209.62:35309), e042378830b94a6e8d9826c011220f98 (127.11.209.61:41765)
20:02:05.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:05.911041 12566 leader_election.cc:334] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ec327abf8690480e82493bad20d856ad (127.11.209.62:35309): Network error: Client connection negotiation failed: client connection to 127.11.209.62:35309: connect: Connection refused (error 111)
20:02:05.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:05.914197 12563 leader_election.cc:334] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e042378830b94a6e8d9826c011220f98 (127.11.209.61:41765): Network error: Client connection negotiation failed: client connection to 127.11.209.61:41765: connect: Connection refused (error 111)
20:02:05.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:05.915004 12563 leader_election.cc:302] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 48846ff3bd7e4bd999c2d47fcfe534a9; no voters: e042378830b94a6e8d9826c011220f98, ec327abf8690480e82493bad20d856ad
20:02:05.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:05.916021 12670 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:02:05.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:05.998476 12622 rpc_server.cc:307] RPC server started. Bound to: 127.11.209.62:35309
20:02:05.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:05.998584 12727 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.11.209.62:35309 every 8 connection(s)
20:02:06.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:06.001634 12622 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-0/data/info.pb
20:02:06.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:06.011950 12100 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskBduYyb/build/tsan/bin/kudu as pid 12622
20:02:06.012 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:02:06.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:06.015537 12729 sys_catalog.cc:263] Verifying existing consensus state
20:02:06.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260811 20:02:06.016548 12729 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
20:02:06.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:06.017122 12622 master.cc:537] Master@127.11.209.62:35309 shutting down...
20:02:06.090 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.11.209.61:41765(127.11.209.61:41765)] connecting to peer
20:02:06.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:06.121556 12622 master.cc:552] Master@127.11.209.62:35309 shutdown complete.
20:02:06.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskBduYyb/test-tmp/mini-kudu-cluster762766219299956452/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
20:02:06.172 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.11.209.60:33937(127.11.209.60:33937)] connecting to peer
20:02:06.178 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.11.209.62:35309(127.11.209.62:35309)] connecting to peer
20:02:06.239 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765
20:02:06.240 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.209.61:41765(127.11.209.61:41765)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765
20:02:06.242 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.11.209.60:33937(127.11.209.60:33937)] Successfully connected to peer
20:02:06.245 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.209.61:41765: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765
20:02:06.246 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
20:02:06.246 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.209.62:35309(127.11.209.62:35309)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
20:02:06.247 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.209.62:35309: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
20:02:06.344 [WARN - kudu-nio-1] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
20:02:06.356 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.11.209.60
20:02:06.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 20:02:06 dist-test-slave-9gcw krb5kdc[12116](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786478519, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.11.209.60@KRBTEST.COM
20:02:06.634 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x67233fc9, L:/127.0.0.1:34380 - R:/127.11.209.60:33937] using SASL/GSSAPI
20:02:06.651 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309; will retry
20:02:06.660 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.209.61:41765(127.11.209.61:41765)] connecting to peer
20:02:06.666 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.209.62:35309(127.11.209.62:35309)] connecting to peer
20:02:06.670 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
20:02:06.671 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.209.62:35309(127.11.209.62:35309)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
20:02:06.672 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.209.62:35309: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
20:02:06.673 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765
20:02:06.673 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.209.61:41765(127.11.209.61:41765)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765
20:02:06.674 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.209.61:41765: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765
20:02:06.677 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309; will retry
20:02:06.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:06.690060 12638 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:02:06.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:06.690372 12638 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:02:06.699 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.209.61:41765(127.11.209.61:41765)] connecting to peer
20:02:06.700 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.209.62:35309(127.11.209.62:35309)] connecting to peer
20:02:06.702 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
20:02:06.702 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.209.62:35309(127.11.209.62:35309)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
20:02:06.703 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.209.62:35309: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
20:02:06.704 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765
20:02:06.704 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.209.61:41765(127.11.209.61:41765)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765
20:02:06.706 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.209.61:41765: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765
20:02:06.709 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309; will retry
20:02:06.719 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.209.61:41765(127.11.209.61:41765)] connecting to peer
20:02:06.720 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.209.62:35309(127.11.209.62:35309)] connecting to peer
20:02:06.723 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
20:02:06.723 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.209.62:35309(127.11.209.62:35309)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
20:02:06.724 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.209.62:35309: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
20:02:06.725 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765
20:02:06.725 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.209.61:41765(127.11.209.61:41765)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765
20:02:06.726 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.209.61:41765: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765
20:02:06.729 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309; will retry
20:02:06.738 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.209.61:41765(127.11.209.61:41765)] connecting to peer
20:02:06.740 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.209.62:35309(127.11.209.62:35309)] connecting to peer
20:02:06.742 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
20:02:06.742 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765
20:02:06.743 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.209.62:35309(127.11.209.62:35309)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
20:02:06.743 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.209.61:41765(127.11.209.61:41765)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765
20:02:06.743 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.209.62:35309: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
20:02:06.744 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.209.61:41765: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765
20:02:06.745 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309; will retry
20:02:06.779 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.209.61:41765(127.11.209.61:41765)] connecting to peer
20:02:06.781 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.209.62:35309(127.11.209.62:35309)] connecting to peer
20:02:06.784 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
20:02:06.784 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.209.62:35309(127.11.209.62:35309)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
20:02:06.789 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.209.62:35309: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
20:02:06.791 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765
20:02:06.793 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.209.61:41765(127.11.209.61:41765)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765
20:02:06.793 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.209.61:41765: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765
20:02:06.794 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309; will retry
20:02:06.799 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.209.61:41765(127.11.209.61:41765)] connecting to peer
20:02:06.800 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.209.62:35309(127.11.209.62:35309)] connecting to peer
20:02:06.805 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
20:02:06.805 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.209.62:35309(127.11.209.62:35309)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
20:02:06.806 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.209.62:35309: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
20:02:06.806 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765
20:02:06.806 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.209.61:41765(127.11.209.61:41765)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765
20:02:06.807 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.209.61:41765: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765
20:02:06.809 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309; will retry
20:02:06.859 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.209.61:41765(127.11.209.61:41765)] connecting to peer
20:02:06.862 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.209.62:35309(127.11.209.62:35309)] connecting to peer
20:02:06.863 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765
20:02:06.863 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.209.61:41765(127.11.209.61:41765)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765
20:02:06.865 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.209.61:41765: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765
20:02:06.865 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
20:02:06.865 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.209.62:35309(127.11.209.62:35309)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
20:02:06.866 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.209.62:35309: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
20:02:06.867 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309; will retry
20:02:06.959 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.209.61:41765(127.11.209.61:41765)] connecting to peer
20:02:06.961 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.209.62:35309(127.11.209.62:35309)] connecting to peer
20:02:06.967 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
20:02:06.967 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.209.62:35309(127.11.209.62:35309)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
20:02:06.969 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.209.62:35309: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
20:02:06.969 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765
20:02:06.969 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.209.61:41765(127.11.209.61:41765)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765
20:02:06.970 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.209.61:41765: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765
20:02:06.970 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309; will retry
20:02:07.099 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.209.61:41765(127.11.209.61:41765)] connecting to peer
20:02:07.100 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.209.62:35309(127.11.209.62:35309)] connecting to peer
20:02:07.103 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
20:02:07.103 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765
20:02:07.103 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.209.62:35309(127.11.209.62:35309)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
20:02:07.103 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.209.61:41765(127.11.209.61:41765)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765
20:02:07.104 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.209.61:41765: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765
20:02:07.104 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.209.62:35309: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
20:02:07.105 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309; will retry
20:02:07.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:07.694437 12638 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:02:07.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:07.694767 12638 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:02:07.799 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.209.61:41765(127.11.209.61:41765)] connecting to peer
20:02:07.801 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.209.62:35309(127.11.209.62:35309)] connecting to peer
20:02:07.801 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765
20:02:07.801 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.209.61:41765(127.11.209.61:41765)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765
20:02:07.802 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.209.61:41765: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765
20:02:07.803 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
20:02:07.803 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.209.62:35309(127.11.209.62:35309)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
20:02:07.803 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.209.62:35309: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
20:02:07.808 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309; will retry
20:02:08.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:08.175877 12739 raft_consensus.cc:482] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:02:08.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:08.176317 12739 raft_consensus.cc:504] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ec327abf8690480e82493bad20d856ad" member_type: VOTER last_known_addr { host: "127.11.209.62" port: 35309 } } peers { permanent_uuid: "e042378830b94a6e8d9826c011220f98" member_type: VOTER last_known_addr { host: "127.11.209.61" port: 41765 } } peers { permanent_uuid: "48846ff3bd7e4bd999c2d47fcfe534a9" member_type: VOTER last_known_addr { host: "127.11.209.60" port: 33937 } }
20:02:08.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:08.178303 12739 leader_election.cc:288] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ec327abf8690480e82493bad20d856ad (127.11.209.62:35309), e042378830b94a6e8d9826c011220f98 (127.11.209.61:41765)
20:02:08.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:08.184655 12566 leader_election.cc:334] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ec327abf8690480e82493bad20d856ad (127.11.209.62:35309): Network error: Client connection negotiation failed: client connection to 127.11.209.62:35309: connect: Connection refused (error 111)
20:02:08.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:08.185177 12563 leader_election.cc:334] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e042378830b94a6e8d9826c011220f98 (127.11.209.61:41765): Network error: Client connection negotiation failed: client connection to 127.11.209.61:41765: connect: Connection refused (error 111)
20:02:08.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:08.185570 12563 leader_election.cc:302] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 48846ff3bd7e4bd999c2d47fcfe534a9; no voters: e042378830b94a6e8d9826c011220f98, ec327abf8690480e82493bad20d856ad
20:02:08.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:08.186240 12739 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:02:08.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:08.698154 12638 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:02:08.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:08.698554 12638 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:02:08.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:08.710736 12536 master_runner.cc:186] Error getting master registration for 127.11.209.62:35309: Network error: Client connection negotiation failed: client connection to 127.11.209.62:35309: connect: Connection refused (error 111),
20:02:08.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:08.711102 12536 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:02:08.999 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.209.61:41765(127.11.209.61:41765)] connecting to peer
20:02:09.001 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.209.62:35309(127.11.209.62:35309)] connecting to peer
20:02:09.003 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765
20:02:09.003 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
20:02:09.003 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.209.61:41765(127.11.209.61:41765)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765
20:02:09.003 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.209.62:35309(127.11.209.62:35309)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
20:02:09.003 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.209.61:41765: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765
20:02:09.003 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.209.62:35309: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
20:02:09.005 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309; will retry
20:02:09.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:09.705250 12638 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:02:09.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:09.706118 12638 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:02:10.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:10.709960 12638 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:02:10.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:10.710325 12638 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:02:11.419 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.209.61:41765(127.11.209.61:41765)] connecting to peer
20:02:11.421 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.209.62:35309(127.11.209.62:35309)] connecting to peer
20:02:11.424 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
20:02:11.424 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.209.62:35309(127.11.209.62:35309)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
20:02:11.424 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.209.62:35309: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
20:02:11.425 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765
20:02:11.425 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.209.61:41765(127.11.209.61:41765)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765
20:02:11.425 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.209.61:41765: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765
20:02:11.427 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309; will retry
20:02:11.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:11.713876 12638 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:02:11.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:11.714238 12638 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:02:12.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:12.092247 12750 raft_consensus.cc:482] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:02:12.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:12.093294 12750 raft_consensus.cc:504] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ec327abf8690480e82493bad20d856ad" member_type: VOTER last_known_addr { host: "127.11.209.62" port: 35309 } } peers { permanent_uuid: "e042378830b94a6e8d9826c011220f98" member_type: VOTER last_known_addr { host: "127.11.209.61" port: 41765 } } peers { permanent_uuid: "48846ff3bd7e4bd999c2d47fcfe534a9" member_type: VOTER last_known_addr { host: "127.11.209.60" port: 33937 } }
20:02:12.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:12.095799 12750 leader_election.cc:288] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ec327abf8690480e82493bad20d856ad (127.11.209.62:35309), e042378830b94a6e8d9826c011220f98 (127.11.209.61:41765)
20:02:12.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:12.099146 12563 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.11.209.61:41765: connect: Connection refused (error 111) [suppressed 5 similar messages]
20:02:12.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:12.102843 12566 leader_election.cc:334] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ec327abf8690480e82493bad20d856ad (127.11.209.62:35309): Network error: Client connection negotiation failed: client connection to 127.11.209.62:35309: connect: Connection refused (error 111)
20:02:12.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:12.103304 12563 leader_election.cc:334] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e042378830b94a6e8d9826c011220f98 (127.11.209.61:41765): Network error: Client connection negotiation failed: client connection to 127.11.209.61:41765: connect: Connection refused (error 111)
20:02:12.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:12.103966 12563 leader_election.cc:302] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 48846ff3bd7e4bd999c2d47fcfe534a9; no voters: e042378830b94a6e8d9826c011220f98, ec327abf8690480e82493bad20d856ad
20:02:12.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:12.104808 12750 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:02:12.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:12.718043 12638 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:02:12.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:12.718356 12638 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:02:13.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 20:02:13 dist-test-slave-9gcw krb5kdc[12116](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786478533, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.209.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:02:13.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:13.352798 12557 init.cc:295] Successfully reacquired a new kerberos TGT
20:02:13.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:13.713853 12536 master_runner.cc:186] Error getting master registration for 127.11.209.62:35309: Network error: Client connection negotiation failed: client connection to 127.11.209.62:35309: connect: Connection refused (error 111),
20:02:13.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:13.714311 12536 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:02:13.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:13.721323 12638 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:02:13.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:13.721570 12638 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:02:14.399 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.209.61:41765(127.11.209.61:41765)] connecting to peer
20:02:14.401 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.209.62:35309(127.11.209.62:35309)] connecting to peer
20:02:14.403 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765
20:02:14.403 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
20:02:14.403 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.209.61:41765(127.11.209.61:41765)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765
20:02:14.403 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.209.62:35309(127.11.209.62:35309)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
20:02:14.404 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.209.62:35309: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
20:02:14.404 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.209.61:41765: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765
20:02:14.406 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309; will retry
20:02:14.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:14.724525 12638 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:02:14.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:14.724857 12638 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:02:15.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:15.728160 12638 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:02:15.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:15.728543 12638 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:02:16.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:16.177759 12082 debug-util.cc:398] Leaking SignalData structure 0x7b08000b9e40 after lost signal to thread 11928
20:02:16.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:16.178838 12082 debug-util.cc:398] Leaking SignalData structure 0x7b08000b9ec0 after lost signal to thread 12085
20:02:16.259 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.209.61:41765(127.11.209.61:41765)] connecting to peer
20:02:16.260 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.209.62:35309(127.11.209.62:35309)] connecting to peer
20:02:16.266 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765
20:02:16.266 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.209.61:41765(127.11.209.61:41765)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765
20:02:16.266 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.209.61:41765: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765
20:02:16.266 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
20:02:16.267 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.209.62:35309(127.11.209.62:35309)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
20:02:16.267 [INFO - pool-6-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.209.62:35309: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
20:02:16.268 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309; will retry
20:02:16.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:16.449532 12759 raft_consensus.cc:482] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:02:16.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:16.450068 12759 raft_consensus.cc:504] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ec327abf8690480e82493bad20d856ad" member_type: VOTER last_known_addr { host: "127.11.209.62" port: 35309 } } peers { permanent_uuid: "e042378830b94a6e8d9826c011220f98" member_type: VOTER last_known_addr { host: "127.11.209.61" port: 41765 } } peers { permanent_uuid: "48846ff3bd7e4bd999c2d47fcfe534a9" member_type: VOTER last_known_addr { host: "127.11.209.60" port: 33937 } }
20:02:16.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:16.451972 12759 leader_election.cc:288] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ec327abf8690480e82493bad20d856ad (127.11.209.62:35309), e042378830b94a6e8d9826c011220f98 (127.11.209.61:41765)
20:02:16.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:16.459132 12566 leader_election.cc:334] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ec327abf8690480e82493bad20d856ad (127.11.209.62:35309): Network error: Client connection negotiation failed: client connection to 127.11.209.62:35309: connect: Connection refused (error 111)
20:02:16.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:16.459589 12563 leader_election.cc:334] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e042378830b94a6e8d9826c011220f98 (127.11.209.61:41765): Network error: Client connection negotiation failed: client connection to 127.11.209.61:41765: connect: Connection refused (error 111)
20:02:16.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:16.459935 12563 leader_election.cc:302] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 48846ff3bd7e4bd999c2d47fcfe534a9; no voters: e042378830b94a6e8d9826c011220f98, ec327abf8690480e82493bad20d856ad
20:02:16.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:16.460480 12759 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:02:16.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:16.732075 12638 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:02:16.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:16.732509 12638 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:02:17.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:17.735915 12638 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:02:17.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:17.736217 12638 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:02:18.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:18.717011 12536 master_runner.cc:186] Error getting master registration for 127.11.209.62:35309: Network error: Client connection negotiation failed: client connection to 127.11.209.62:35309: connect: Connection refused (error 111),
20:02:18.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:18.717489 12536 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:02:18.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:18.739519 12638 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:02:18.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:18.739832 12638 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:02:19.299 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.209.61:41765(127.11.209.61:41765)] connecting to peer
20:02:19.301 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.209.62:35309(127.11.209.62:35309)] connecting to peer
20:02:19.303 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765
20:02:19.303 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.209.61:41765(127.11.209.61:41765)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765
20:02:19.304 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.209.61:41765: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765
20:02:19.304 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
20:02:19.304 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.209.62:35309(127.11.209.62:35309)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
20:02:19.304 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.209.62:35309: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
20:02:19.305 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309; will retry
20:02:19.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:19.568948 12774 raft_consensus.cc:482] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:02:19.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:19.569384 12774 raft_consensus.cc:504] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ec327abf8690480e82493bad20d856ad" member_type: VOTER last_known_addr { host: "127.11.209.62" port: 35309 } } peers { permanent_uuid: "e042378830b94a6e8d9826c011220f98" member_type: VOTER last_known_addr { host: "127.11.209.61" port: 41765 } } peers { permanent_uuid: "48846ff3bd7e4bd999c2d47fcfe534a9" member_type: VOTER last_known_addr { host: "127.11.209.60" port: 33937 } }
20:02:19.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:19.570892 12774 leader_election.cc:288] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ec327abf8690480e82493bad20d856ad (127.11.209.62:35309), e042378830b94a6e8d9826c011220f98 (127.11.209.61:41765)
20:02:19.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:19.573007 12563 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.11.209.61:41765: connect: Connection refused (error 111) [suppressed 3 similar messages]
20:02:19.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:19.576051 12563 leader_election.cc:334] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e042378830b94a6e8d9826c011220f98 (127.11.209.61:41765): Network error: Client connection negotiation failed: client connection to 127.11.209.61:41765: connect: Connection refused (error 111)
20:02:19.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:19.576512 12566 leader_election.cc:334] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ec327abf8690480e82493bad20d856ad (127.11.209.62:35309): Network error: Client connection negotiation failed: client connection to 127.11.209.62:35309: connect: Connection refused (error 111)
20:02:19.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:19.576841 12566 leader_election.cc:302] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 48846ff3bd7e4bd999c2d47fcfe534a9; no voters: e042378830b94a6e8d9826c011220f98, ec327abf8690480e82493bad20d856ad
20:02:19.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:19.577521 12774 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:02:19.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:19.743294 12638 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:02:19.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:19.743577 12638 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:02:20.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:20.746943 12638 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:02:20.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:20.747216 12638 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:02:21.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:21.750329 12638 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:02:21.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:21.750586 12638 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:02:22.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:22.754222 12638 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:02:22.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:22.754565 12638 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:02:22.959 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.209.61:41765(127.11.209.61:41765)] connecting to peer
20:02:22.963 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765
20:02:22.963 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.209.61:41765(127.11.209.61:41765)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765
20:02:22.963 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.209.61:41765: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765
20:02:22.961 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.209.62:35309(127.11.209.62:35309)] connecting to peer
20:02:22.966 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
20:02:22.966 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.209.62:35309(127.11.209.62:35309)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
20:02:22.966 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.209.62:35309: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
20:02:22.966 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309; will retry
20:02:23.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 20:02:23 dist-test-slave-9gcw krb5kdc[12116](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786478543, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.209.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:02:23.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:23.355947 12557 init.cc:295] Successfully reacquired a new kerberos TGT
20:02:23.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:23.721098 12536 master_runner.cc:186] Error getting master registration for 127.11.209.62:35309: Network error: Client connection negotiation failed: client connection to 127.11.209.62:35309: connect: Connection refused (error 111),
20:02:23.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:23.721711 12536 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:02:23.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:23.758230 12638 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:02:23.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:23.758582 12638 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:02:24.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:24.761906 12638 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:02:24.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:24.762188 12638 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:02:25.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:25.765506 12638 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:02:25.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:25.765779 12638 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:02:26.579 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.209.61:41765(127.11.209.61:41765)] connecting to peer
20:02:26.582 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.209.62:35309(127.11.209.62:35309)] connecting to peer
20:02:26.585 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765
20:02:26.586 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.209.61:41765(127.11.209.61:41765)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765
20:02:26.586 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.209.61:41765: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765
20:02:26.587 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
20:02:26.587 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.209.62:35309(127.11.209.62:35309)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
20:02:26.587 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.209.62:35309: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
20:02:26.588 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309; will retry
20:02:26.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:26.768821 12638 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:02:26.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:26.769135 12638 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:02:27.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:27.772315 12638 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:02:27.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:27.772584 12638 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:02:28.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:28.680044 12082 debug-util.cc:398] Leaking SignalData structure 0x7b08000b9960 after lost signal to thread 11928
20:02:28.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:28.680886 12082 debug-util.cc:398] Leaking SignalData structure 0x7b08000afec0 after lost signal to thread 12085
20:02:28.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:28.724577 12536 master_runner.cc:186] Error getting master registration for 127.11.209.62:35309: Network error: Client connection negotiation failed: client connection to 127.11.209.62:35309: connect: Connection refused (error 111),
20:02:28.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:28.724984 12536 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:02:28.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:28.775739 12638 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:02:28.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:28.776044 12638 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:02:29.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:29.779105 12638 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:02:29.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:29.779378 12638 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:02:29.939 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.209.61:41765(127.11.209.61:41765)] connecting to peer
20:02:29.940 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.209.62:35309(127.11.209.62:35309)] connecting to peer
20:02:29.942 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765
20:02:29.942 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
20:02:29.942 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.209.61:41765(127.11.209.61:41765)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765
20:02:29.943 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.209.62:35309(127.11.209.62:35309)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
20:02:29.943 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.209.61:41765: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765
20:02:29.943 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.209.62:35309: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
20:02:29.944 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309; will retry
20:02:30.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:30.782652 12638 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:02:30.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:30.783057 12638 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:02:31.499 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.209.61:41765(127.11.209.61:41765)] connecting to peer
20:02:31.501 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.209.62:35309(127.11.209.62:35309)] connecting to peer
20:02:31.502 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
20:02:31.502 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765
20:02:31.503 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.209.62:35309(127.11.209.62:35309)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
20:02:31.503 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.209.61:41765(127.11.209.61:41765)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765
20:02:31.503 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.209.62:35309: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
20:02:31.503 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.209.61:41765: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765
20:02:31.505 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309; will retry
20:02:31.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:31.786284 12638 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:02:31.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:31.786600 12638 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:02:32.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:32.790843 12638 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:02:32.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:32.791262 12638 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:02:33.239 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.209.61:41765(127.11.209.61:41765)] connecting to peer
20:02:33.240 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.209.62:35309(127.11.209.62:35309)] connecting to peer
20:02:33.241 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
20:02:33.241 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765
20:02:33.241 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.209.62:35309(127.11.209.62:35309)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
20:02:33.241 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.209.61:41765(127.11.209.61:41765)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765
20:02:33.241 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.209.62:35309: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
20:02:33.242 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.209.61:41765: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765
20:02:33.245 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309; will retry
20:02:33.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:33.322647 12799 raft_consensus.cc:482] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:02:33.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:33.323227 12799 raft_consensus.cc:504] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ec327abf8690480e82493bad20d856ad" member_type: VOTER last_known_addr { host: "127.11.209.62" port: 35309 } } peers { permanent_uuid: "e042378830b94a6e8d9826c011220f98" member_type: VOTER last_known_addr { host: "127.11.209.61" port: 41765 } } peers { permanent_uuid: "48846ff3bd7e4bd999c2d47fcfe534a9" member_type: VOTER last_known_addr { host: "127.11.209.60" port: 33937 } }
20:02:33.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:33.325323 12799 leader_election.cc:288] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ec327abf8690480e82493bad20d856ad (127.11.209.62:35309), e042378830b94a6e8d9826c011220f98 (127.11.209.61:41765)
20:02:33.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:33.328083 12566 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.11.209.62:35309: connect: Connection refused (error 111) [suppressed 1 similar messages]
20:02:33.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:33.328080 12563 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.11.209.61:41765: connect: Connection refused (error 111)
20:02:33.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:33.331238 12563 leader_election.cc:334] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e042378830b94a6e8d9826c011220f98 (127.11.209.61:41765): Network error: Client connection negotiation failed: client connection to 127.11.209.61:41765: connect: Connection refused (error 111)
20:02:33.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:33.331686 12566 leader_election.cc:334] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ec327abf8690480e82493bad20d856ad (127.11.209.62:35309): Network error: Client connection negotiation failed: client connection to 127.11.209.62:35309: connect: Connection refused (error 111)
20:02:33.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:33.332026 12566 leader_election.cc:302] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 48846ff3bd7e4bd999c2d47fcfe534a9; no voters: e042378830b94a6e8d9826c011220f98, ec327abf8690480e82493bad20d856ad
20:02:33.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:33.332679 12799 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:02:33.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 11 20:02:33 dist-test-slave-9gcw krb5kdc[12116](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786478553, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.209.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:02:33.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:33.358688 12557 init.cc:295] Successfully reacquired a new kerberos TGT
20:02:33.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:33.727288 12536 master_runner.cc:186] Error getting master registration for 127.11.209.62:35309: Network error: Client connection negotiation failed: client connection to 127.11.209.62:35309: connect: Connection refused (error 111),
20:02:33.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:33.727685 12536 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:02:33.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:33.794137 12638 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 48846ff3bd7e4bd999c2d47fcfe534a9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:02:33.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260811 20:02:33.794392 12638 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:02:34.339 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.209.61:41765(127.11.209.61:41765)] connecting to peer
20:02:34.340 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.209.62:35309(127.11.209.62:35309)] connecting to peer
20:02:34.341 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765
20:02:34.341 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
20:02:34.341 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.209.61:41765(127.11.209.61:41765)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765
20:02:34.341 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.209.62:35309(127.11.209.62:35309)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
20:02:34.342 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.209.61:41765: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765
20:02:34.342 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.209.62:35309: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
20:02:34.344 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309; will retry
20:02:34.349 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=22, TimeoutTracker(timeout=30000, elapsed=28327), Traces: [0ms] refreshing cache from master, [54ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.61:41765, [148ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.60:33937, [154ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.62:35309, [217ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.61:41765: Network error: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [223ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.62:35309: Network error: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [623ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.60:33937: OK, [628ms] delaying RPC due to: Service unavailable: Master config (127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [635ms] refreshing cache from master, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.61:41765, [639ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.60:33937, [642ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.62:35309, [647ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.62:35309: Network error: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [650ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.61:41765: Network error: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [652ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.60:33937: OK, [654ms] delaying RPC due to: Service unavailable: Master config (127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [674ms] refreshing cache from master, [674ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.61:41765, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.60:33937, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.62:35309, [679ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.62:35309: Network error: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [681ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.61:41765: Network error: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [683ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.60:33937: OK, [685ms] delaying RPC due to: Service unavailable: Master config (127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [694ms] refreshing cache from master, [694ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.61:41765, [695ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.60:33937, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.62:35309, [699ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.62:35309: Network error: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [701ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.61:41765: Network error: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [704ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.60:33937: OK, [705ms] delaying RPC due to: Service unavailable: Master config (127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [714ms] refreshing cache from master, [714ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.61:41765, [715ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.60:33937, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.62:35309, [719ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.61:41765: Network error: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [719ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.62:35309: Network error: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [721ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.60:33937: OK, [722ms] delaying RPC due to: Service unavailable: Master config (127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [754ms] refreshing cache from master, [755ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.61:41765, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.60:33937, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.62:35309, [760ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.62:35309: Network error: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [766ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.60:33937: OK, [769ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.61:41765: Network error: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [770ms] delaying RPC due to: Service unavailable: Master config (127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [774ms] refreshing cache from master, [775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.61:41765, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.60:33937, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.62:35309, [781ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.62:35309: Network error: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [783ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.61:41765: Network error: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [785ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.60:33937: OK, [786ms] delaying RPC due to: Service unavailable: Master config (127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [834ms] refreshing cache from master, [835ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.61:41765, [836ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.60:33937, [837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.62:35309, [839ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.61:41765: Network error: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [841ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.62:35309: Network error: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [843ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.60:33937: OK, [844ms] delaying RPC due to: Service unavailable: Master config (127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [934ms] refreshing cache from master, [935ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.61:41765, [936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.60:33937, [937ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.62:35309, [944ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.62:35309: Network error: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [944ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.60:33937: OK, [945ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.61:41765: Network error: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [946ms] delaying RPC due to: Service unavailable: Master config (127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [1074ms] refreshing cache from master, [1074ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.61:41765, [1076ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.60:33937, [1076ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.62:35309, [1079ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.62:35309: Network error: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [1079ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.61:41765: Network error: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [1081ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.60:33937: OK, [1082ms] delaying RPC due to: Service unavailable: Master config (127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [1774ms] refreshing cache from master, [1775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.61:41765, [1776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.60:33937, [1777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.62:35309, [1777ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.61:41765: Network error: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [1779ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.62:35309: Network error: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [1783ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.60:33937: OK, [1784ms] delaying RPC due to: Service unavailable: Master config (127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [2975ms] refreshing cache from master, [2975ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.61:41765, [2976ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.60:33937, [2977ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.62:35309, [2979ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.61:41765: Network error: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [2979ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.62:35309: Network error: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [2981ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.60:33937: OK, [2981ms] delaying RPC due to: Service unavailable: Master config (127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [5395ms] refreshing cache from master, [5395ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.61:41765, [5396ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.60:33937, [5397ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.62:35309, [5400ms] too many traces: truncated at 100 traces, deferred=Deferred@528495565(state=PENDING, result=null, callback=(continuation of Deferred@883830224 after retry RPC after error@1567250312) -> (continuation of Deferred@806514350 after retry RPC after error@781331770) -> (continuation of Deferred@1796691349 after retry RPC after error@1691257192) -> (continuation of Deferred@1191898177 after retry RPC after error@2026824295) -> (continuation of Deferred@690268950 after retry RPC after error@959570579) -> (continuation of Deferred@692680976 after retry RPC after error@647719812) -> (continuation of Deferred@720050266 after retry RPC after error@486036206) -> (continuation of Deferred@2124102040 after retry RPC after error@52542084) -> (continuation of Deferred@1912232818 after retry RPC after error@801835137) -> (continuation of Deferred@138130201 after retry RPC after error@113302784) -> (continuation of Deferred@633485153 after retry RPC after error@1387969624) -> (continuation of Deferred@130842221 after retry RPC after error@132418377) -> (continuation of Deferred@866766315 after retry RPC after error@888598458) -> (continuation of Deferred@1353564255 after retry RPC after error@1932572608) -> (continuation of Deferred@244050648 after retry RPC after error@226713484) -> (continuation of Deferred@856873307 after retry RPC after error@1164637774) -> (continuation of Deferred@350619921 after retry RPC after error@51268414) -> (continuation of Deferred@653492741 after retry RPC after error@876202003) -> (continuation of Deferred@280228100 after retry RPC after error@1628770199) -> (continuation of Deferred@1540114970 after retry RPC after error@567240736) -> (continuation of Deferred@788235980 after retry RPC after error@1734040883), errback=(continuation of Deferred@883830224 after retry RPC after error@1567250312) -> (continuation of Deferred@806514350 after retry RPC after error@781331770) -> (continuation of Deferred@1796691349 after retry RPC after error@1691257192) -> (continuation of Deferred@1191898177 after retry RPC after error@2026824295) -> (continuation of Deferred@690268950 after retry RPC after error@959570579) -> (continuation of Deferred@692680976 after retry RPC after error@647719812) -> (continuation of Deferred@720050266 after retry RPC after error@486036206) -> (continuation of Deferred@2124102040 after retry RPC after error@52542084) -> (continuation of Deferred@1912232818 after retry RPC after error@801835137) -> (continuation of Deferred@138130201 after retry RPC after error@113302784) -> (continuation of Deferred@633485153 after retry RPC after error@1387969624) -> (continuation of Deferred@130842221 after retry RPC after error@132418377) -> (continuation of Deferred@866766315 after retry RPC after error@888598458) -> (continuation of Deferred@1353564255 after retry RPC after error@1932572608) -> (continuation of Deferred@244050648 after retry RPC after error@226713484) -> (continuation of Deferred@856873307 after retry RPC after error@1164637774) -> (continuation of Deferred@350619921 after retry RPC after error@51268414) -> (continuation of Deferred@653492741 after retry RPC after error@876202003) -> (continuation of Deferred@280228100 after retry RPC after error@1628770199) -> (continuation of Deferred@1540114970 after retry RPC after error@567240736) -> (continuation of Deferred@788235980 after retry RPC after error@1734040883)))
20:02:34.354 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.11.209.60:33937(127.11.209.60:33937)] disconnecting while in state READY
20:02:34.356 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.11.209.62:35309(127.11.209.62:35309)] disconnecting while in state TERMINATED
20:02:34.356 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.11.209.61:41765(127.11.209.61:41765)] disconnecting while in state TERMINATED
20:02:34.362 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
20:02:34.366 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
20:02:34.367 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.11.209.60:33937(127.11.209.60:33937)] handling channelInactive
20:02:34.367 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.209.60:33937(127.11.209.60:33937)] cleaning up while in state READY due to: connection closed
20:02:34.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:34.372565 12100 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskBduYyb/build/tsan/bin/kudu with pid 12536
20:02:35.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-11T20:02:35Z chronyd exiting
20:02:36.656 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
20:02:36.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:36.657954 11227 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskBduYyb/build/tsan/bin/kudu with pid 11567
20:02:36.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:36.699723 11227 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskBduYyb/build/tsan/bin/kudu with pid 11755
20:02:36.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:36.731880 11227 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskBduYyb/build/tsan/bin/kudu with pid 11927
20:02:36.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:36.769389 11227 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskBduYyb/build/tsan/bin/kudu with pid 11243
20:02:36.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:36.809058 11227 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskBduYyb/build/tsan/bin/kudu with pid 11330
20:02:36.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260811 20:02:36.849426 11227 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskBduYyb/build/tsan/bin/kudu with pid 11419
20:02:38.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-11T20:02:38Z chronyd exiting
20:02:39.072 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=22, TimeoutTracker(timeout=30000, elapsed=28327), Traces: [0ms] refreshing cache from master, [54ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.61:41765, [148ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.60:33937, [154ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.62:35309, [217ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.61:41765: Network error: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [223ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.62:35309: Network error: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [623ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.60:33937: OK, [628ms] delaying RPC due to: Service unavailable: Master config (127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [635ms] refreshing cache from master, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.61:41765, [639ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.60:33937, [642ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.62:35309, [647ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.62:35309: Network error: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [650ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.61:41765: Network error: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [652ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.60:33937: OK, [654ms] delaying RPC due to: Service unavailable: Master config (127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [674ms] refreshing cache from master, [674ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.61:41765, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.60:33937, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.62:35309, [679ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.62:35309: Network error: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [681ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.61:41765: Network error: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [683ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.60:33937: OK, [685ms] delaying RPC due to: Service unavailable: Master config (127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [694ms] refreshing cache from master, [694ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.61:41765, [695ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.60:33937, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.62:35309, [699ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.62:35309: Network error: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [701ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.61:41765: Network error: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [704ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.60:33937: OK, [705ms] delaying RPC due to: Service unavailable: Master config (127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [714ms] refreshing cache from master, [714ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.61:41765, [715ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.60:33937, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.62:35309, [719ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.61:41765: Network error: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [719ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.62:35309: Network error: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [721ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.60:33937: OK, [722ms] delaying RPC due to: Service unavailable: Master config (127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [754ms] refreshing cache from master, [755ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.61:41765, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.60:33937, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.62:35309, [760ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.62:35309: Network error: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [766ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.60:33937: OK, [769ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.61:41765: Network error: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [770ms] delaying RPC due to: Service unavailable: Master config (127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [774ms] refreshing cache from master, [775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.61:41765, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.60:33937, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.62:35309, [781ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.62:35309: Network error: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [783ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.61:41765: Network error: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [785ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.60:33937: OK, [786ms] delaying RPC due to: Service unavailable: Master config (127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [834ms] refreshing cache from master, [835ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.61:41765, [836ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.60:33937, [837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.62:35309, [839ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.61:41765: Network error: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [841ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.62:35309: Network error: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [843ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.60:33937: OK, [844ms] delaying RPC due to: Service unavailable: Master config (127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [934ms] refreshing cache from master, [935ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.61:41765, [936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.60:33937, [937ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.62:35309, [944ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.62:35309: Network error: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [944ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.60:33937: OK, [945ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.61:41765: Network error: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [946ms] delaying RPC due to: Service unavailable: Master config (127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [1074ms] refreshing cache from master, [1074ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.61:41765, [1076ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.60:33937, [1076ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.62:35309, [1079ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.62:35309: Network error: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [1079ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.61:41765: Network error: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [1081ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.60:33937: OK, [1082ms] delaying RPC due to: Service unavailable: Master config (127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [1774ms] refreshing cache from master, [1775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.61:41765, [1776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.60:33937, [1777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.62:35309, [1777ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.61:41765: Network error: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [1779ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.62:35309: Network error: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [1783ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.60:33937: OK, [1784ms] delaying RPC due to: Service unavailable: Master config (127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [2975ms] refreshing cache from master, [2975ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.61:41765, [2976ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.60:33937, [2977ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.62:35309, [2979ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.61:41765: Network error: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [2979ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.62:35309: Network error: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [2981ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.60:33937: OK, [2981ms] delaying RPC due to: Service unavailable: Master config (127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [5395ms] refreshing cache from master, [5395ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.61:41765, [5396ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.60:33937, [5397ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.62:35309, [5400ms] too many traces: truncated at 100 traces, deferred=Deferred@528495565(state=PENDING, result=null, callback=(continuation of Deferred@883830224 after retry RPC after error@1567250312) -> (continuation of Deferred@806514350 after retry RPC after error@781331770) -> (continuation of Deferred@1796691349 after retry RPC after error@1691257192) -> (continuation of Deferred@1191898177 after retry RPC after error@2026824295) -> (continuation of Deferred@690268950 after retry RPC after error@959570579) -> (continuation of Deferred@692680976 after retry RPC after error@647719812) -> (continuation of Deferred@720050266 after retry RPC after error@486036206) -> (continuation of Deferred@2124102040 after retry RPC after error@52542084) -> (continuation of Deferred@1912232818 after retry RPC after error@801835137) -> (continuation of Deferred@138130201 after retry RPC after error@113302784) -> (continuation of Deferred@633485153 after retry RPC after error@1387969624) -> (continuation of Deferred@130842221 after retry RPC after error@132418377) -> (continuation of Deferred@866766315 after retry RPC after error@888598458) -> (continuation of Deferred@1353564255 after retry RPC after error@1932572608) -> (continuation of Deferred@244050648 after retry RPC after error@226713484) -> (continuation of Deferred@856873307 after retry RPC after error@1164637774) -> (continuation of Deferred@350619921 after retry RPC after error@51268414) -> (continuation of Deferred@653492741 after retry RPC after error@876202003) -> (continuation of Deferred@280228100 after retry RPC after error@1628770199) -> (continuation of Deferred@1540114970 after retry RPC after error@567240736) -> (continuation of Deferred@788235980 after retry RPC after error@1734040883), errback=(continuation of Deferred@883830224 after retry RPC after error@1567250312) -> (continuation of Deferred@806514350 after retry RPC after error@781331770) -> (continuation of Deferred@1796691349 after retry RPC after error@1691257192) -> (continuation of Deferred@1191898177 after retry RPC after error@2026824295) -> (continuation of Deferred@690268950 after retry RPC after error@959570579) -> (continuation of Deferred@692680976 after retry RPC after error@647719812) -> (continuation of Deferred@720050266 after retry RPC after error@486036206) -> (continuation of Deferred@2124102040 after retry RPC after error@52542084) -> (continuation of Deferred@1912232818 after retry RPC after error@801835137) -> (continuation of Deferred@138130201 after retry RPC after error@113302784) -> (continuation of Deferred@633485153 after retry RPC after error@1387969624) -> (continuation of Deferred@130842221 after retry RPC after error@132418377) -> (continuation of Deferred@866766315 after retry RPC after error@888598458) -> (continuation of Deferred@1353564255 after retry RPC after error@1932572608) -> (continuation of Deferred@244050648 after retry RPC after error@226713484) -> (continuation of Deferred@856873307 after retry RPC after error@1164637774) -> (continuation of Deferred@350619921 after retry RPC after error@51268414) -> (continuation of Deferred@653492741 after retry RPC after error@876202003) -> (continuation of Deferred@280228100 after retry RPC after error@1628770199) -> (continuation of Deferred@1540114970 after retry RPC after error@567240736) -> (continuation of Deferred@788235980 after retry RPC after error@1734040883)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.11.209.62:35309
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=22, TimeoutTracker(timeout=30000, elapsed=28327), Traces: [0ms] refreshing cache from master, [54ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.61:41765, [148ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.60:33937, [154ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.62:35309, [217ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.61:41765: Network error: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [223ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.62:35309: Network error: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [623ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.60:33937: OK, [628ms] delaying RPC due to: Service unavailable: Master config (127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [635ms] refreshing cache from master, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.61:41765, [639ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.60:33937, [642ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.62:35309, [647ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.62:35309: Network error: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [650ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.61:41765: Network error: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [652ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.60:33937: OK, [654ms] delaying RPC due to: Service unavailable: Master config (127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [674ms] refreshing cache from master, [674ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.61:41765, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.60:33937, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.62:35309, [679ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.62:35309: Network error: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [681ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.61:41765: Network error: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [683ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.60:33937: OK, [685ms] delaying RPC due to: Service unavailable: Master config (127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [694ms] refreshing cache from master, [694ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.61:41765, [695ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.60:33937, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.62:35309, [699ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.62:35309: Network error: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [701ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.61:41765: Network error: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [704ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.60:33937: OK, [705ms] delaying RPC due to: Service unavailable: Master config (127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [714ms] refreshing cache from master, [714ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.61:41765, [715ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.60:33937, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.62:35309, [719ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.61:41765: Network error: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [719ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.62:35309: Network error: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [721ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.60:33937: OK, [722ms] delaying RPC due to: Service unavailable: Master config (127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [754ms] refreshing cache from master, [755ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.61:41765, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.60:33937, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.62:35309, [760ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.62:35309: Network error: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [766ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.60:33937: OK, [769ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.61:41765: Network error: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [770ms] delaying RPC due to: Service unavailable: Master config (127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [774ms] refreshing cache from master, [775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.61:41765, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.60:33937, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.62:35309, [781ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.62:35309: Network error: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [783ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.61:41765: Network error: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [785ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.60:33937: OK, [786ms] delaying RPC due to: Service unavailable: Master config (127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [834ms] refreshing cache from master, [835ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.61:41765, [836ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.60:33937, [837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.62:35309, [839ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.61:41765: Network error: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [841ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.62:35309: Network error: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [843ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.60:33937: OK, [844ms] delaying RPC due to: Service unavailable: Master config (127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [934ms] refreshing cache from master, [935ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.61:41765, [936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.60:33937, [937ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.62:35309, [944ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.62:35309: Network error: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [944ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.60:33937: OK, [945ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.61:41765: Network error: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [946ms] delaying RPC due to: Service unavailable: Master config (127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [1074ms] refreshing cache from master, [1074ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.61:41765, [1076ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.60:33937, [1076ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.62:35309, [1079ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.62:35309: Network error: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [1079ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.61:41765: Network error: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [1081ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.60:33937: OK, [1082ms] delaying RPC due to: Service unavailable: Master config (127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [1774ms] refreshing cache from master, [1775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.61:41765, [1776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.60:33937, [1777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.62:35309, [1777ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.61:41765: Network error: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [1779ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.62:35309: Network error: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [1783ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.60:33937: OK, [1784ms] delaying RPC due to: Service unavailable: Master config (127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [2975ms] refreshing cache from master, [2975ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.61:41765, [2976ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.60:33937, [2977ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.62:35309, [2979ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.61:41765: Network error: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [2979ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.62:35309: Network error: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [2981ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.60:33937: OK, [2981ms] delaying RPC due to: Service unavailable: Master config (127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [5395ms] refreshing cache from master, [5395ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.61:41765, [5396ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.60:33937, [5397ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.62:35309, [5400ms] too many traces: truncated at 100 traces, deferred=Deferred@528495565(state=PENDING, result=null, callback=(continuation of Deferred@883830224 after retry RPC after error@1567250312) -> (continuation of Deferred@806514350 after retry RPC after error@781331770) -> (continuation of Deferred@1796691349 after retry RPC after error@1691257192) -> (continuation of Deferred@1191898177 after retry RPC after error@2026824295) -> (continuation of Deferred@690268950 after retry RPC after error@959570579) -> (continuation of Deferred@692680976 after retry RPC after error@647719812) -> (continuation of Deferred@720050266 after retry RPC after error@486036206) -> (continuation of Deferred@2124102040 after retry RPC after error@52542084) -> (continuation of Deferred@1912232818 after retry RPC after error@801835137) -> (continuation of Deferred@138130201 after retry RPC after error@113302784) -> (continuation of Deferred@633485153 after retry RPC after error@1387969624) -> (continuation of Deferred@130842221 after retry RPC after error@132418377) -> (continuation of Deferred@866766315 after retry RPC after error@888598458) -> (continuation of Deferred@1353564255 after retry RPC after error@1932572608) -> (continuation of Deferred@244050648 after retry RPC after error@226713484) -> (continuation of Deferred@856873307 after retry RPC after error@1164637774) -> (continuation of Deferred@350619921 after retry RPC after error@51268414) -> (continuation of Deferred@653492741 after retry RPC after error@876202003) -> (continuation of Deferred@280228100 after retry RPC after error@1628770199) -> (continuation of Deferred@1540114970 after retry RPC after error@567240736) -> (continuation of Deferred@788235980 after retry RPC after error@1734040883), errback=(continuation of Deferred@883830224 after retry RPC after error@1567250312) -> (continuation of Deferred@806514350 after retry RPC after error@781331770) -> (continuation of Deferred@1796691349 after retry RPC after error@1691257192) -> (continuation of Deferred@1191898177 after retry RPC after error@2026824295) -> (continuation of Deferred@690268950 after retry RPC after error@959570579) -> (continuation of Deferred@692680976 after retry RPC after error@647719812) -> (continuation of Deferred@720050266 after retry RPC after error@486036206) -> (continuation of Deferred@2124102040 after retry RPC after error@52542084) -> (continuation of Deferred@1912232818 after retry RPC after error@801835137) -> (continuation of Deferred@138130201 after retry RPC after error@113302784) -> (continuation of Deferred@633485153 after retry RPC after error@1387969624) -> (continuation of Deferred@130842221 after retry RPC after error@132418377) -> (continuation of Deferred@866766315 after retry RPC after error@888598458) -> (continuation of Deferred@1353564255 after retry RPC after error@1932572608) -> (continuation of Deferred@244050648 after retry RPC after error@226713484) -> (continuation of Deferred@856873307 after retry RPC after error@1164637774) -> (continuation of Deferred@350619921 after retry RPC after error@51268414) -> (continuation of Deferred@653492741 after retry RPC after error@876202003) -> (continuation of Deferred@280228100 after retry RPC after error@1628770199) -> (continuation of Deferred@1540114970 after retry RPC after error@567240736) -> (continuation of Deferred@788235980 after retry RPC after error@1734040883)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.11.209.62:35309
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
20:02:39.478 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
20:02:39.479 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=22, TimeoutTracker(timeout=30000, elapsed=28327), Traces: [0ms] refreshing cache from master, [54ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.61:41765, [148ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.60:33937, [154ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.62:35309, [217ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.61:41765: Network error: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [223ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.62:35309: Network error: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [623ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.60:33937: OK, [628ms] delaying RPC due to: Service unavailable: Master config (127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [635ms] refreshing cache from master, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.61:41765, [639ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.60:33937, [642ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.62:35309, [647ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.62:35309: Network error: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [650ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.61:41765: Network error: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [652ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.60:33937: OK, [654ms] delaying RPC due to: Service unavailable: Master config (127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [674ms] refreshing cache from master, [674ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.61:41765, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.60:33937, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.62:35309, [679ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.62:35309: Network error: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [681ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.61:41765: Network error: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [683ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.60:33937: OK, [685ms] delaying RPC due to: Service unavailable: Master config (127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [694ms] refreshing cache from master, [694ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.61:41765, [695ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.60:33937, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.62:35309, [699ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.62:35309: Network error: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [701ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.61:41765: Network error: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [704ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.60:33937: OK, [705ms] delaying RPC due to: Service unavailable: Master config (127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [714ms] refreshing cache from master, [714ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.61:41765, [715ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.60:33937, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.62:35309, [719ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.61:41765: Network error: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [719ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.62:35309: Network error: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [721ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.60:33937: OK, [722ms] delaying RPC due to: Service unavailable: Master config (127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [754ms] refreshing cache from master, [755ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.61:41765, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.60:33937, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.62:35309, [760ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.62:35309: Network error: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [766ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.60:33937: OK, [769ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.61:41765: Network error: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [770ms] delaying RPC due to: Service unavailable: Master config (127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [774ms] refreshing cache from master, [775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.61:41765, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.60:33937, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.62:35309, [781ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.62:35309: Network error: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [783ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.61:41765: Network error: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [785ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.60:33937: OK, [786ms] delaying RPC due to: Service unavailable: Master config (127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [834ms] refreshing cache from master, [835ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.61:41765, [836ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.60:33937, [837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.62:35309, [839ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.61:41765: Network error: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [841ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.62:35309: Network error: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [843ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.60:33937: OK, [844ms] delaying RPC due to: Service unavailable: Master config (127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [934ms] refreshing cache from master, [935ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.61:41765, [936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.60:33937, [937ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.62:35309, [944ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.62:35309: Network error: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [944ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.60:33937: OK, [945ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.61:41765: Network error: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [946ms] delaying RPC due to: Service unavailable: Master config (127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [1074ms] refreshing cache from master, [1074ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.61:41765, [1076ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.60:33937, [1076ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.62:35309, [1079ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.62:35309: Network error: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [1079ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.61:41765: Network error: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [1081ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.60:33937: OK, [1082ms] delaying RPC due to: Service unavailable: Master config (127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [1774ms] refreshing cache from master, [1775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.61:41765, [1776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.60:33937, [1777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.62:35309, [1777ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.61:41765: Network error: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [1779ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.62:35309: Network error: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [1783ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.60:33937: OK, [1784ms] delaying RPC due to: Service unavailable: Master config (127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [2975ms] refreshing cache from master, [2975ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.61:41765, [2976ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.60:33937, [2977ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.62:35309, [2979ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.61:41765: Network error: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [2979ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.62:35309: Network error: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [2981ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.60:33937: OK, [2981ms] delaying RPC due to: Service unavailable: Master config (127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [5395ms] refreshing cache from master, [5395ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.61:41765, [5396ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.60:33937, [5397ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.62:35309, [5400ms] too many traces: truncated at 100 traces, deferred=Deferred@528495565(state=PENDING, result=null, callback=(continuation of Deferred@883830224 after retry RPC after error@1567250312) -> (continuation of Deferred@806514350 after retry RPC after error@781331770) -> (continuation of Deferred@1796691349 after retry RPC after error@1691257192) -> (continuation of Deferred@1191898177 after retry RPC after error@2026824295) -> (continuation of Deferred@690268950 after retry RPC after error@959570579) -> (continuation of Deferred@692680976 after retry RPC after error@647719812) -> (continuation of Deferred@720050266 after retry RPC after error@486036206) -> (continuation of Deferred@2124102040 after retry RPC after error@52542084) -> (continuation of Deferred@1912232818 after retry RPC after error@801835137) -> (continuation of Deferred@138130201 after retry RPC after error@113302784) -> (continuation of Deferred@633485153 after retry RPC after error@1387969624) -> (continuation of Deferred@130842221 after retry RPC after error@132418377) -> (continuation of Deferred@866766315 after retry RPC after error@888598458) -> (continuation of Deferred@1353564255 after retry RPC after error@1932572608) -> (continuation of Deferred@244050648 after retry RPC after error@226713484) -> (continuation of Deferred@856873307 after retry RPC after error@1164637774) -> (continuation of Deferred@350619921 after retry RPC after error@51268414) -> (continuation of Deferred@653492741 after retry RPC after error@876202003) -> (continuation of Deferred@280228100 after retry RPC after error@1628770199) -> (continuation of Deferred@1540114970 after retry RPC after error@567240736) -> (continuation of Deferred@788235980 after retry RPC after error@1734040883), errback=(continuation of Deferred@883830224 after retry RPC after error@1567250312) -> (continuation of Deferred@806514350 after retry RPC after error@781331770) -> (continuation of Deferred@1796691349 after retry RPC after error@1691257192) -> (continuation of Deferred@1191898177 after retry RPC after error@2026824295) -> (continuation of Deferred@690268950 after retry RPC after error@959570579) -> (continuation of Deferred@692680976 after retry RPC after error@647719812) -> (continuation of Deferred@720050266 after retry RPC after error@486036206) -> (continuation of Deferred@2124102040 after retry RPC after error@52542084) -> (continuation of Deferred@1912232818 after retry RPC after error@801835137) -> (continuation of Deferred@138130201 after retry RPC after error@113302784) -> (continuation of Deferred@633485153 after retry RPC after error@1387969624) -> (continuation of Deferred@130842221 after retry RPC after error@132418377) -> (continuation of Deferred@866766315 after retry RPC after error@888598458) -> (continuation of Deferred@1353564255 after retry RPC after error@1932572608) -> (continuation of Deferred@244050648 after retry RPC after error@226713484) -> (continuation of Deferred@856873307 after retry RPC after error@1164637774) -> (continuation of Deferred@350619921 after retry RPC after error@51268414) -> (continuation of Deferred@653492741 after retry RPC after error@876202003) -> (continuation of Deferred@280228100 after retry RPC after error@1628770199) -> (continuation of Deferred@1540114970 after retry RPC after error@567240736) -> (continuation of Deferred@788235980 after retry RPC after error@1734040883)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.11.209.62:35309
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=22, TimeoutTracker(timeout=30000, elapsed=28327), Traces: [0ms] refreshing cache from master, [54ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.61:41765, [148ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.60:33937, [154ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.62:35309, [217ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.61:41765: Network error: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [223ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.62:35309: Network error: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [623ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.60:33937: OK, [628ms] delaying RPC due to: Service unavailable: Master config (127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [635ms] refreshing cache from master, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.61:41765, [639ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.60:33937, [642ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.62:35309, [647ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.62:35309: Network error: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [650ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.61:41765: Network error: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [652ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.60:33937: OK, [654ms] delaying RPC due to: Service unavailable: Master config (127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [674ms] refreshing cache from master, [674ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.61:41765, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.60:33937, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.62:35309, [679ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.62:35309: Network error: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [681ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.61:41765: Network error: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [683ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.60:33937: OK, [685ms] delaying RPC due to: Service unavailable: Master config (127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [694ms] refreshing cache from master, [694ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.61:41765, [695ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.60:33937, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.62:35309, [699ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.62:35309: Network error: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [701ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.61:41765: Network error: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [704ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.60:33937: OK, [705ms] delaying RPC due to: Service unavailable: Master config (127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [714ms] refreshing cache from master, [714ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.61:41765, [715ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.60:33937, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.62:35309, [719ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.61:41765: Network error: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [719ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.62:35309: Network error: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [721ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.60:33937: OK, [722ms] delaying RPC due to: Service unavailable: Master config (127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [754ms] refreshing cache from master, [755ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.61:41765, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.60:33937, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.62:35309, [760ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.62:35309: Network error: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [766ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.60:33937: OK, [769ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.61:41765: Network error: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [770ms] delaying RPC due to: Service unavailable: Master config (127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [774ms] refreshing cache from master, [775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.61:41765, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.60:33937, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.62:35309, [781ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.62:35309: Network error: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [783ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.61:41765: Network error: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [785ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.60:33937: OK, [786ms] delaying RPC due to: Service unavailable: Master config (127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [834ms] refreshing cache from master, [835ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.61:41765, [836ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.60:33937, [837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.62:35309, [839ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.61:41765: Network error: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [841ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.62:35309: Network error: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [843ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.60:33937: OK, [844ms] delaying RPC due to: Service unavailable: Master config (127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [934ms] refreshing cache from master, [935ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.61:41765, [936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.60:33937, [937ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.62:35309, [944ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.62:35309: Network error: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [944ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.60:33937: OK, [945ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.61:41765: Network error: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [946ms] delaying RPC due to: Service unavailable: Master config (127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [1074ms] refreshing cache from master, [1074ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.61:41765, [1076ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.60:33937, [1076ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.62:35309, [1079ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.62:35309: Network error: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [1079ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.61:41765: Network error: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [1081ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.60:33937: OK, [1082ms] delaying RPC due to: Service unavailable: Master config (127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [1774ms] refreshing cache from master, [1775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.61:41765, [1776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.60:33937, [1777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.62:35309, [1777ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.61:41765: Network error: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [1779ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.62:35309: Network error: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [1783ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.60:33937: OK, [1784ms] delaying RPC due to: Service unavailable: Master config (127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [2975ms] refreshing cache from master, [2975ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.61:41765, [2976ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.60:33937, [2977ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.62:35309, [2979ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.61:41765: Network error: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765, [2979ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.62:35309: Network error: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [2981ms] Sub RPC ConnectToMaster: received response from server master-127.11.209.60:33937: OK, [2981ms] delaying RPC due to: Service unavailable: Master config (127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309, [5395ms] refreshing cache from master, [5395ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.61:41765, [5396ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.60:33937, [5397ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.209.62:35309, [5400ms] too many traces: truncated at 100 traces, deferred=Deferred@528495565(state=PENDING, result=null, callback=(continuation of Deferred@883830224 after retry RPC after error@1567250312) -> (continuation of Deferred@806514350 after retry RPC after error@781331770) -> (continuation of Deferred@1796691349 after retry RPC after error@1691257192) -> (continuation of Deferred@1191898177 after retry RPC after error@2026824295) -> (continuation of Deferred@690268950 after retry RPC after error@959570579) -> (continuation of Deferred@692680976 after retry RPC after error@647719812) -> (continuation of Deferred@720050266 after retry RPC after error@486036206) -> (continuation of Deferred@2124102040 after retry RPC after error@52542084) -> (continuation of Deferred@1912232818 after retry RPC after error@801835137) -> (continuation of Deferred@138130201 after retry RPC after error@113302784) -> (continuation of Deferred@633485153 after retry RPC after error@1387969624) -> (continuation of Deferred@130842221 after retry RPC after error@132418377) -> (continuation of Deferred@866766315 after retry RPC after error@888598458) -> (continuation of Deferred@1353564255 after retry RPC after error@1932572608) -> (continuation of Deferred@244050648 after retry RPC after error@226713484) -> (continuation of Deferred@856873307 after retry RPC after error@1164637774) -> (continuation of Deferred@350619921 after retry RPC after error@51268414) -> (continuation of Deferred@653492741 after retry RPC after error@876202003) -> (continuation of Deferred@280228100 after retry RPC after error@1628770199) -> (continuation of Deferred@1540114970 after retry RPC after error@567240736) -> (continuation of Deferred@788235980 after retry RPC after error@1734040883), errback=(continuation of Deferred@883830224 after retry RPC after error@1567250312) -> (continuation of Deferred@806514350 after retry RPC after error@781331770) -> (continuation of Deferred@1796691349 after retry RPC after error@1691257192) -> (continuation of Deferred@1191898177 after retry RPC after error@2026824295) -> (continuation of Deferred@690268950 after retry RPC after error@959570579) -> (continuation of Deferred@692680976 after retry RPC after error@647719812) -> (continuation of Deferred@720050266 after retry RPC after error@486036206) -> (continuation of Deferred@2124102040 after retry RPC after error@52542084) -> (continuation of Deferred@1912232818 after retry RPC after error@801835137) -> (continuation of Deferred@138130201 after retry RPC after error@113302784) -> (continuation of Deferred@633485153 after retry RPC after error@1387969624) -> (continuation of Deferred@130842221 after retry RPC after error@132418377) -> (continuation of Deferred@866766315 after retry RPC after error@888598458) -> (continuation of Deferred@1353564255 after retry RPC after error@1932572608) -> (continuation of Deferred@244050648 after retry RPC after error@226713484) -> (continuation of Deferred@856873307 after retry RPC after error@1164637774) -> (continuation of Deferred@350619921 after retry RPC after error@51268414) -> (continuation of Deferred@653492741 after retry RPC after error@876202003) -> (continuation of Deferred@280228100 after retry RPC after error@1628770199) -> (continuation of Deferred@1540114970 after retry RPC after error@567240736) -> (continuation of Deferred@788235980 after retry RPC after error@1734040883)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.11.209.61:41765,127.11.209.60:33937,127.11.209.62:35309) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.61:41765(127.11.209.61:41765): Connection refused: /127.11.209.61:41765,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.209.62:35309(127.11.209.62:35309): Connection refused: /127.11.209.62:35309
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.11.209.62:35309
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)